You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/04/11 09:13:12 UTC
[1/9] kylin git commit: minor, refine GTCubeStorageQueryRequest
Repository: kylin
Updated Branches:
refs/heads/master a36288142 -> e9ec116c2
minor, refine GTCubeStorageQueryRequest
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d216e223
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d216e223
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d216e223
Branch: refs/heads/master
Commit: d216e223b0d1ef67aaa93aa820a11f04ac69cb2e
Parents: 9250d9b
Author: lidongsjtu <li...@apache.org>
Authored: Fri Mar 31 22:57:52 2017 +0800
Committer: lidongsjtu <li...@apache.org>
Committed: Fri Mar 31 22:57:52 2017 +0800
----------------------------------------------------------------------
.../kylin/storage/gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../storage/gtrecord/GTCubeStorageQueryRequest.java | 12 +++++++++++-
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d216e223/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
index 065fd6e..083080c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
@@ -150,7 +150,7 @@ public abstract class GTCubeStorageQueryBase implements IStorageQuery {
logger.info("Cuboid identified: cube={}, cuboidId={}, groupsD={}, filterD={}, limitPushdown={}, storageAggr={}", cubeInstance.getName(), cuboid.getId(), groupsD, filterColumnD, context.getFinalPushDownLimit(), context.isNeedStorageAggregation());
- return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, metrics, filterD, context);
+ return new GTCubeStorageQueryRequest(cuboid, dimensionsD, groupsD, filterColumnD, metrics, filterD, context);
}
protected abstract String getGTStorage();
http://git-wip-us.apache.org/repos/asf/kylin/blob/d216e223/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
----------------------------------------------------------------------
diff --git a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
index c4d81d2..68f755c 100644
--- a/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
+++ b/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryRequest.java
@@ -31,14 +31,16 @@ public class GTCubeStorageQueryRequest implements Serializable {
private Cuboid cuboid;
private Set<TblColRef> dimensions;
private Set<TblColRef> groups;
+ private Set<TblColRef> filterCols;
private Set<FunctionDesc> metrics;
private TupleFilter filter;
private StorageContext context;
- public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<FunctionDesc> metrics, TupleFilter filter, StorageContext context) {
+ public GTCubeStorageQueryRequest(Cuboid cuboid, Set<TblColRef> dimensions, Set<TblColRef> groups, Set<TblColRef> filterCols, Set<FunctionDesc> metrics, TupleFilter filter, StorageContext context) {
this.cuboid = cuboid;
this.dimensions = dimensions;
this.groups = groups;
+ this.filterCols = filterCols;
this.metrics = metrics;
this.filter = filter;
this.context = context;
@@ -91,4 +93,12 @@ public class GTCubeStorageQueryRequest implements Serializable {
public void setContext(StorageContext context) {
this.context = context;
}
+
+ public Set<TblColRef> getFilterCols() {
+ return filterCols;
+ }
+
+ public void setFilterCols(Set<TblColRef> filterCols) {
+ this.filterCols = filterCols;
+ }
}
[4/9] kylin git commit: minor, fix check style error
Posted by li...@apache.org.
minor, fix check style error
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/88e66f3e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/88e66f3e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/88e66f3e
Branch: refs/heads/master
Commit: 88e66f3ea642816c91c964836aa468c758ec65d9
Parents: 266a62a
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Apr 5 18:35:55 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Wed Apr 5 18:36:16 2017 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/88e66f3e/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index 67e811a..bbf3c60 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,12 +19,9 @@
package org.apache.kylin.source.hive;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.IncompleteArgumentException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -61,8 +58,6 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
-import javax.el.MethodNotFoundException;
-
public class HiveMRInput implements IMRInput {
public static String getTableNameForHCat(TableDesc table, boolean isFullTable) {
[8/9] kylin git commit: minor bug fix
Posted by li...@apache.org.
minor bug fix
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/f44f8289
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/f44f8289
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/f44f8289
Branch: refs/heads/master
Commit: f44f828928aaecac3e7dafee69d9443b043b0d4f
Parents: 6b4f70d
Author: Hongbin Ma <ma...@apache.org>
Authored: Wed Apr 5 14:07:33 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Apr 11 14:54:21 2017 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/rest/service/ProjectService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/f44f8289/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
----------------------------------------------------------------------
diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
index e6f546c..8c0cf7a 100644
--- a/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
+++ b/server-base/src/main/java/org/apache/kylin/rest/service/ProjectService.java
@@ -104,7 +104,7 @@ public class ProjectService extends BasicService {
return projects.subList(coffset, coffset + climit);
}
- @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
+ @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION') or hasPermission(#project, 'MANAGEMENT')")
public void deleteProject(String projectName, ProjectInstance project) throws IOException {
getProjectManager().dropProject(projectName);
[5/9] kylin git commit: KYLIN 2513 SUM measure return type should not
overflow
Posted by li...@apache.org.
KYLIN 2513 SUM measure return type should not overflow
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/023fa238
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/023fa238
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/023fa238
Branch: refs/heads/master
Commit: 023fa2387f9685efc64cc21a0fcc7b2e4457d9e7
Parents: 88e66f3
Author: chenzhx <34...@qq.com>
Authored: Thu Apr 6 15:34:14 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Thu Apr 6 15:34:14 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeMeasures.js | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/023fa238/webapp/app/js/controllers/cubeMeasures.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeMeasures.js b/webapp/app/js/controllers/cubeMeasures.js
index 856e74f..73a6cd0 100644
--- a/webapp/app/js/controllers/cubeMeasures.js
+++ b/webapp/app/js/controllers/cubeMeasures.js
@@ -370,10 +370,10 @@ KylinApp.controller('CubeMeasuresCtrl', function ($scope, $modal,MetaModel,cubes
if(colType==="smallint"||colType==="int"||colType==="bigint"||colType==="integer"){
$scope.newMeasure.function.returntype= 'bigint';
}else{
- if(colType.indexOf('decimal')!=-1){
- $scope.newMeasure.function.returntype= colType;
+ if(colType.indexOf('decimal')!=-1||colType==="double"||colType==="float"){
+ $scope.newMeasure.function.returntype= 'decimal(19,4)';
}else{
- $scope.newMeasure.function.returntype= 'decimal';
+ $scope.newMeasure.function.returntype= 'decimal(14,0)';
}
}
break;
[3/9] kylin git commit: KYLIN-2531 convert old tableExt jackson to
new one instead of removing it
Posted by li...@apache.org.
KYLIN-2531 convert old tableExt jackson to new one instead of removing it
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/266a62a5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/266a62a5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/266a62a5
Branch: refs/heads/master
Commit: 266a62a5ab6b0b39b08b79d3f3d4d1948e111247
Parents: 38fe432
Author: Cheng Wang <ch...@kyligence.io>
Authored: Wed Apr 5 14:43:52 2017 +0800
Committer: Dong Li <li...@apache.org>
Committed: Wed Apr 5 14:46:12 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/metadata/MetadataManager.java | 64 +++++++++++++++-----
.../kylin/metadata/MetadataManagerTest.java | 41 +++++++++++--
2 files changed, 85 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
index 9427ace..f17983f 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/MetadataManager.java
@@ -19,8 +19,10 @@
package org.apache.kylin.metadata;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -30,8 +32,10 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.JsonSerializer;
+import org.apache.kylin.common.persistence.RawResource;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.common.persistence.Serializer;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.metadata.cachesync.Broadcaster;
import org.apache.kylin.metadata.cachesync.Broadcaster.Event;
import org.apache.kylin.metadata.cachesync.CaseInsensitiveStringCache;
@@ -47,6 +51,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
/**
* Serves (and caches) metadata for Kylin instance.
@@ -207,11 +212,7 @@ public class MetadataManager {
result.setUuid(UUID.randomUUID().toString());
result.setLastModified(0);
result.init();
- try {
- saveTableExt(result);
- } catch (IOException ex) {
- logger.warn("Failed to save TableExt", ex);
- }
+ srcTableExdMap.put(result.getName(), result);
}
return result;
}
@@ -224,9 +225,14 @@ public class MetadataManager {
tableExt.init();
String path = tableExt.getResourcePath();
- getStore().putResource(path, tableExt, TABLE_EXT_SERIALIZER);
- srcTableExdMap.put(tableExt.getName(), tableExt);
+ ResourceStore store = getStore();
+
+ TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+ if (t != null && t.getName() == null)
+ store.deleteResource(path);
+
+ store.putResource(path, tableExt, TABLE_EXT_SERIALIZER);
}
public void removeTableExt(String tableName) throws IOException {
@@ -387,22 +393,52 @@ public class MetadataManager {
private TableExtDesc reloadTableExtAt(String path) throws IOException {
ResourceStore store = getStore();
TableExtDesc t = store.getResource(path, TableExtDesc.class, TABLE_EXT_SERIALIZER);
+
if (t == null) {
return null;
}
+
+ // convert old tableExt json to new one
+ if (t.getName() == null) {
+ t = convertOldTableExtToNewer(path);
+ }
+
t.init();
- String name = t.getName();
+ srcTableExdMap.putLocal(t.getName(), t);
+ return t;
+ }
- // remove old json
- if (name == null) {
- getStore().deleteResource(path);
- return null;
+ private TableExtDesc convertOldTableExtToNewer(String path) throws IOException {
+ Map<String, String> attrs = Maps.newHashMap();
+
+ ResourceStore store = getStore();
+ RawResource res = store.getResource(path);
+
+ InputStream is = res.inputStream;
+
+ try {
+ attrs.putAll(JsonUtil.readValue(is, HashMap.class));
+ } finally {
+ if (is != null)
+ is.close();
}
- srcTableExdMap.putLocal(name, t);
+ String cardinality = attrs.get(MetadataConstants.TABLE_EXD_CARDINALITY);
- return t;
+ // parse table identity from file name
+ String file = path;
+ if (file.indexOf("/") > -1) {
+ file = file.substring(file.lastIndexOf("/") + 1);
+ }
+ String tableIdentity = file.substring(0, file.length() - MetadataConstants.FILE_SURFIX.length()).toUpperCase();
+ TableExtDesc result = new TableExtDesc();
+ result.setName(tableIdentity);
+ result.setUuid(UUID.randomUUID().toString());
+ result.setLastModified(0);
+ result.init();
+ result.setCardinality(cardinality);
+ return result;
}
private void reloadAllExternalFilter() throws IOException {
http://git-wip-us.apache.org/repos/asf/kylin/blob/266a62a5/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
index a861ee1..eb04287 100644
--- a/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
+++ b/core-metadata/src/test/java/org/apache/kylin/metadata/MetadataManagerTest.java
@@ -23,10 +23,16 @@ import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
+import org.apache.kylin.common.util.JsonUtil;
import org.apache.kylin.common.util.LocalFileMetadataTestCase;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.JoinTableDesc;
@@ -51,7 +57,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
public void after() throws Exception {
this.cleanupTestMetadata();
}
-
+
@Test
public void testCiModel() {
MetadataManager mgr = getInstance(getTestConfig());
@@ -59,7 +65,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
DataModelDesc im = mgr.getDataModelDesc("ci_inner_join_model");
assertSnowflakeQuality(lm);
assertSnowflakeQuality(im);
-
+
// check inner/left models are identical apart from the left/inner difference
assertEquals(lm.getJoinTables().length, im.getJoinTables().length);
for (int i = 0, n = im.getJoinTables().length; i < n; i++) {
@@ -73,7 +79,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
assertTrue(lt.getJoin().isLeftJoin());
assertTrue(it.getJoin().isInnerJoin());
}
-
+
assertEquals(lm.getDimensions().size(), im.getDimensions().size());
for (int i = 0, n = im.getDimensions().size(); i < n; i++) {
ModelDimensionDesc ld = lm.getDimensions().get(i);
@@ -81,10 +87,10 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
assertEquals(ld.getTable(), id.getTable());
assertArrayEquals(ld.getColumns(), id.getColumns());
}
-
+
assertArrayEquals(lm.getMetrics(), im.getMetrics());
}
-
+
private void assertSnowflakeQuality(DataModelDesc model) {
Assert.assertNotNull(model);
try {
@@ -93,7 +99,7 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
} catch (IllegalArgumentException ex) {
// excepted
}
-
+
Assert.assertNotNull(model.findTable("BUYER_COUNTRY"));
Assert.assertNotNull(model.findTable("SELLER_COUNTRY"));
Assert.assertNotNull(model.findColumn("BUYER_COUNTRY.NAME"));
@@ -148,4 +154,27 @@ public class MetadataManagerTest extends LocalFileMetadataTestCase {
getInstance(getTestConfig()).removeTableExt("TEST.TEST_TABLE");
}
+
+ @Test
+ public void testTableExtCompatibility() throws IOException {
+ String tableName = "TEST.TEST_TABLE";
+ Map<String, String> oldTableExt = new HashMap<>();
+ oldTableExt.put(MetadataConstants.TABLE_EXD_CARDINALITY, "1,2,3,4");
+ mockUpOldTableExtJson(tableName, oldTableExt);
+ TableExtDesc tableExtDesc = getInstance(getTestConfig()).getTableExt(tableName);
+ Assert.assertEquals("1,2,3,4,", tableExtDesc.getCardinality());
+ getInstance(getTestConfig()).removeTableExt(tableName);
+ }
+
+ private void mockUpOldTableExtJson(String tableId, Map<String, String> tableExdProperties) throws IOException {
+ String path = TableDesc.concatExdResourcePath(tableId);
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ JsonUtil.writeValueIndent(os, tableExdProperties);
+ os.flush();
+ InputStream is = new ByteArrayInputStream(os.toByteArray());
+ getStore().putResource(path, is, System.currentTimeMillis());
+ os.close();
+ is.close();
+ }
}
[2/9] kylin git commit: KYLIN-2532 make Hive flat step more extensible
Posted by li...@apache.org.
KYLIN-2532 make Hive flat step more extensible
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/38fe432d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/38fe432d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/38fe432d
Branch: refs/heads/master
Commit: 38fe432d914d0bde8a0ddda50a6867ea42b5ccbf
Parents: d216e22
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Apr 5 13:12:00 2017 +0800
Committer: Roger Shi <ro...@gmail.com>
Committed: Wed Apr 5 00:20:55 2017 -0500
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfigBase.java | 8 ++++++++
.../org/apache/kylin/job/JoinedFlatTable.java | 18 +++++++++++++++++-
.../org/apache/kylin/source/hive/HiveMRInput.java | 5 +++++
3 files changed, 30 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
index 760b7da..4361242 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java
@@ -448,6 +448,14 @@ abstract public class KylinConfigBase implements Serializable {
return Integer.parseInt(getOptional("kylin.job.error-record-threshold", "0"));
}
+ public boolean isAdvancedFlatTableUsed() {
+ return Boolean.parseBoolean(getOptional("kylin.job.use-advanced-flat-table", "false"));
+ }
+
+ public String getAdvancedFlatTableClass() {
+ return getOptional("kylin.job.advanced-flat-table.class");
+ }
+
// ============================================================================
// SOURCE.HIVE
// ============================================================================
http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 9ed563f..7313630 100644
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@ -19,12 +19,15 @@
package org.apache.kylin.job;
import java.io.File;
+import java.lang.reflect.Method;
import java.util.HashSet;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.cube.CubeSegment;
import org.apache.kylin.job.engine.JobEngineConfig;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.IJoinedFlatTableDesc;
@@ -99,9 +102,22 @@ public class JoinedFlatTable {
}
public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc, JobEngineConfig engineConfig) {
+ final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
StringBuilder sql = new StringBuilder();
+
+ if (kylinConfig.isAdvancedFlatTableUsed()) {
+ try {
+ Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
+ Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class);
+ return (String) method.invoke(null, flatDesc, engineConfig);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";").append("\n");
+ sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n");
+
return sql.toString();
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/38fe432d/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index bbf3c60..67e811a 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -19,9 +19,12 @@
package org.apache.kylin.source.hive;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.IncompleteArgumentException;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -58,6 +61,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.collect.Sets;
+import javax.el.MethodNotFoundException;
+
public class HiveMRInput implements IMRInput {
public static String getTableNameForHCat(TableDesc table, boolean isFullTable) {
[9/9] kylin git commit: Merge commit
'a36288142f30cad931881107872adb4ba0a574a3'
Posted by li...@apache.org.
Merge commit 'a36288142f30cad931881107872adb4ba0a574a3'
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/e9ec116c
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/e9ec116c
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/e9ec116c
Branch: refs/heads/master
Commit: e9ec116c247072603a5405ddba425fd349ba670b
Parents: f44f828 a362881
Author: Hongbin Ma <ma...@apache.org>
Authored: Tue Apr 11 17:12:46 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Tue Apr 11 17:12:46 2017 +0800
----------------------------------------------------------------------
build/conf/kylin.properties | 1 +
.../kylin/common/util/HiveCmdBuilder.java | 16 +++-
.../kylin/common/util/HiveCmdBuilderTest.java | 1 +
.../cube/model/CubeJoinedFlatTableDesc.java | 4 +-
.../org/apache/kylin/job/JoinedFlatTable.java | 34 +++----
.../kylin/job/engine/JobEngineConfig.java | 2 +-
.../apache/kylin/job/JoinedFlatTableTest.java | 4 +-
.../metadata/filter/CompareTupleFilter.java | 14 +++
.../filter/FilterOptimizeTransformer.java | 89 ++++++++++++++++++
.../kylin/metadata/model/FunctionDesc.java | 8 +-
.../kylin/metadata/model/ParameterDesc.java | 58 ++++++------
.../apache/kylin/metadata/model/TblColRef.java | 10 ++
.../gtrecord/GTCubeStorageQueryBase.java | 2 +-
.../gtrecord/StorageResponseGTScatter.java | 22 +++--
.../engine/mr/common/HadoopStatusGetter.java | 15 +--
examples/test_case_data/sandbox/hdfs-site.xml | 4 +
.../test_case_data/sandbox/kylin.properties | 1 +
.../src/test/resources/query/sql/query104.sql | 31 +++++++
.../resources/query/sql_casewhen/query55.sql | 22 +++++
.../query/sql_casewhen/query55.sql.disable | 22 -----
.../resources/query/sql_subquery/query15.sql | 29 ++++++
.../resources/query/sql_subquery/query16.sql | 38 ++++++++
.../kylin/query/relnode/OLAPAggregateRel.java | 13 ++-
.../kylin/query/relnode/OLAPFilterRel.java | 25 ++---
.../kylin/query/relnode/OLAPProjectRel.java | 12 +--
.../kylin/query/relnode/OLAPTableScan.java | 2 +-
.../org/apache/kylin/query/udf/ConcatUDF.java | 28 ++++++
.../org/apache/kylin/rest/util/QueryUtil.java | 15 ---
.../apache/kylin/rest/util/QueryUtilTest.java | 6 --
.../apache/kylin/source/hive/HiveMRInput.java | 97 +++++++++-----------
.../kylin/storage/hbase/HBaseConnection.java | 7 +-
.../hbase/util/DeployCoprocessorCLI.java | 1 +
32 files changed, 425 insertions(+), 208 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
----------------------------------------------------------------------
diff --cc core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
index 7313630,5553d34..54c1e61
--- a/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
+++ b/core-job/src/main/java/org/apache/kylin/job/JoinedFlatTable.java
@@@ -19,8 -19,8 +19,9 @@@
package org.apache.kylin.job;
import java.io.File;
+import java.lang.reflect.Method;
import java.util.HashSet;
+ import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@@ -49,18 -46,18 +50,17 @@@ public class JoinedFlatTable
return storageDfsDir + "/" + flatDesc.getTableName();
}
- public static String generateHiveSetStatements(JobEngineConfig engineConfig) {
- public static String generateHiveInitStatements(
- String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) {
++ public static String generateHiveInitStatements(String flatTableDatabase, String kylinHiveFile, Map<String, String> cubeOverrides) {
+
StringBuilder buffer = new StringBuilder();
+ buffer.append("USE ").append(flatTableDatabase).append(";\n");
try {
- File hadoopPropertiesFile = new File(engineConfig.getHiveConfFilePath());
-
- if (hadoopPropertiesFile.exists()) {
+ File file = new File(kylinHiveFile);
+ if (file.exists()) {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
- DocumentBuilder builder;
- Document doc;
- builder = factory.newDocumentBuilder();
- doc = builder.parse(hadoopPropertiesFile);
+ DocumentBuilder builder = factory.newDocumentBuilder();
+ Document doc = builder.parse(file);
NodeList nl = doc.getElementsByTagName("property");
for (int i = 0; i < nl.getLength(); i++) {
String name = doc.getElementsByTagName("name").item(i).getFirstChild().getNodeValue();
@@@ -101,24 -103,8 +106,19 @@@
return ddl.toString();
}
- public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc, JobEngineConfig engineConfig) {
+ public static String generateInsertDataStatement(IJoinedFlatTableDesc flatDesc) {
+ final KylinConfig kylinConfig = ((CubeSegment) flatDesc.getSegment()).getConfig();
- StringBuilder sql = new StringBuilder();
-
+ if (kylinConfig.isAdvancedFlatTableUsed()) {
+ try {
+ Class advancedFlatTable = Class.forName(kylinConfig.getAdvancedFlatTableClass());
+ Method method = advancedFlatTable.getMethod("generateInsertDataStatement", IJoinedFlatTableDesc.class, JobEngineConfig.class);
- return (String) method.invoke(null, flatDesc, engineConfig);
++ return (String) method.invoke(null, flatDesc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
- sql.append(generateHiveSetStatements(engineConfig));
- sql.append("INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n");
-
- return sql.toString();
+ return "INSERT OVERWRITE TABLE " + flatDesc.getTableName() + " " + generateSelectDataStatement(flatDesc) + ";\n";
}
public static String generateSelectDataStatement(IJoinedFlatTableDesc flatDesc) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/core-storage/src/main/java/org/apache/kylin/storage/gtrecord/GTCubeStorageQueryBase.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/e9ec116c/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
[6/9] kylin git commit: KYLIN 2534 column family select mistake
Posted by li...@apache.org.
KYLIN 2534 column family select mistake
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/75630b8d
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/75630b8d
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/75630b8d
Branch: refs/heads/master
Commit: 75630b8d90a900550a55f4a931eedee4b9eb3d21
Parents: 023fa23
Author: chenzhx <34...@qq.com>
Authored: Thu Apr 6 15:42:52 2017 +0800
Committer: chenzhx <34...@qq.com>
Committed: Thu Apr 6 15:42:52 2017 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeAdvanceSetting.js | 21 ++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/75630b8d/webapp/app/js/controllers/cubeAdvanceSetting.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeAdvanceSetting.js b/webapp/app/js/controllers/cubeAdvanceSetting.js
index 5e7de33..d3e2922 100644
--- a/webapp/app/js/controllers/cubeAdvanceSetting.js
+++ b/webapp/app/js/controllers/cubeAdvanceSetting.js
@@ -339,18 +339,19 @@ KylinApp.controller('CubeAdvanceSettingCtrl', function ($scope, $modal,cubeConfi
var allMeasureNames = $scope.getAllMeasureNames();
var tmpColumnFamily = $scope.cubeMetaFrame.hbase_mapping.column_family;
- angular.forEach($scope.cubeMetaFrame.hbase_mapping.column_family, function (colFamily,index1) {
- angular.forEach(colFamily.columns[0].measure_refs, function (measureName, index2) {
- var allIndex = allMeasureNames.indexOf(measureName);
+ for(var j=0;j<$scope.cubeMetaFrame.hbase_mapping.column_family.length; j++) {
+ for (var i=0;i<$scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs.length; i++){
+ var allIndex = allMeasureNames.indexOf($scope.cubeMetaFrame.hbase_mapping.column_family[j].columns[0].measure_refs[i]);
if (allIndex == -1) {
- tmpColumnFamily[index1].columns[0].measure_refs.splice(index2, 1);
+ tmpColumnFamily[j].columns[0].measure_refs.splice(i, 1);
+ i--
}
-
- if (tmpColumnFamily[index1].columns[0].measure_refs == 0) {
- tmpColumnFamily.splice(index1, 1);
- }
- });
- });
+ }
+ if (tmpColumnFamily[j].columns[0].measure_refs.length == 0) {
+ tmpColumnFamily.splice(j, 1);
+ j--
+ }
+ }
$scope.cubeMetaFrame.hbase_mapping.column_family = tmpColumnFamily;
};
[7/9] kylin git commit: minor, remove method fillbackOptimizedColumn
Posted by li...@apache.org.
minor, remove method fillbackOptimizedColumn
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6b4f70d2
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6b4f70d2
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6b4f70d2
Branch: refs/heads/master
Commit: 6b4f70d257e1eb363a7b792cde8f6f59821094a6
Parents: 75630b8
Author: Roger Shi <ro...@hotmail.com>
Authored: Sun Apr 9 21:26:09 2017 +0800
Committer: Roger Shi <ro...@hotmail.com>
Committed: Sun Apr 9 21:26:09 2017 +0800
----------------------------------------------------------------------
.../kylin/query/relnode/OLAPAggregateRel.java | 18 ------------------
1 file changed, 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6b4f70d2/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
index 2c75a14..1378681 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPAggregateRel.java
@@ -345,8 +345,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
private void buildRewriteFieldsAndMetricsColumns() {
- fillbackOptimizedColumn();
-
ColumnRowType inputColumnRowType = ((OLAPRel) getInput()).getColumnRowType();
RelDataTypeFactory typeFactory = getCluster().getTypeFactory();
for (int i = 0; i < this.aggregations.size(); i++) {
@@ -385,22 +383,6 @@ public class OLAPAggregateRel extends Aggregate implements OLAPRel {
}
}
- private void fillbackOptimizedColumn() {
- // some aggcall will be optimized out in sub-query (e.g. tableau generated sql), we need to fill them back
- RelDataType inputAggRow = getInput().getRowType();
- RelDataType outputAggRow = getRowType();
- if (inputAggRow.getFieldCount() != outputAggRow.getFieldCount()) {
- for (RelDataTypeField inputField : inputAggRow.getFieldList()) {
- String inputFieldName = inputField.getName();
- // constant columns(starts with $) should not be added to context.
- if (!inputFieldName.startsWith("$") && outputAggRow.getField(inputFieldName, true, false) == null) {
- TblColRef column = this.columnRowType.getColumnByIndex(inputField.getIndex());
- this.context.metricsColumns.add(column);
- }
- }
- }
- }
-
private boolean noPrecaculatedFieldsAvailable() {
return !this.context.hasPrecalculatedFields() || !RewriteImplementor.needRewrite(this.context);
}