You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/12/04 09:22:09 UTC
[01/10] kylin git commit: KYLIN-1153 Tool for updating CubeDesc
metadata
Repository: kylin
Updated Branches:
refs/heads/2.0-rc b6e3a28ae -> 83288ac56
KYLIN-1153 Tool for updating CubeDesc metadata
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/976d88bc
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/976d88bc
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/976d88bc
Branch: refs/heads/2.0-rc
Commit: 976d88bceb09a38dbd2fb0eb8e915d5866c3adf3
Parents: b6e3a28
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 30 10:44:47 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:13:37 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/util/ClassUtil.java | 1 +
.../job/upgrade/CubeDescSignatureUpdate.java | 117 +++++++++++++++++++
.../hbase/util/DeployCoprocessorCLI.java | 7 +-
3 files changed, 121 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/976d88bc/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
index 93790e6..ce1f014 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/ClassUtil.java
@@ -50,6 +50,7 @@ public class ClassUtil {
classRenameMap.put("org.apache.kylin.job.common.HadoopShellExecutable", "org.apache.kylin.engine.mr.common.HadoopShellExecutable");
classRenameMap.put("org.apache.kylin.job.common.MapReduceExecutable", "org.apache.kylin.engine.mr.common.MapReduceExecutable");
classRenameMap.put("org.apache.kylin.job.cube.CubingJob", "org.apache.kylin.engine.mr.CubingJob");
+ classRenameMap.put("org.apache.kylin.job.invertedindex.IIJob", "org.apache.kylin.engine.mr.invertedindex.IIJob");
classRenameMap.put("org.apache.kylin.job.cube.GarbageCollectionStep", "org.apache.kylin.storage.hbase.steps.DeprecatedGCStep");
classRenameMap.put("org.apache.kylin.job.cube.MergeDictionaryStep", "org.apache.kylin.engine.mr.steps.MergeDictionaryStep");
classRenameMap.put("org.apache.kylin.job.cube.UpdateCubeInfoAfterBuildStep", "org.apache.kylin.engine.mr.steps.UpdateCubeInfoAfterBuildStep");
http://git-wip-us.apache.org/repos/asf/kylin/blob/976d88bc/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
new file mode 100644
index 0000000..178f1b0
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.job.upgrade;
+
+import com.google.common.collect.Lists;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.model.CubeDesc;
+
+import java.util.List;
+
+/**
+ * Created by dongli on 11/17/15.
+ */
+public class CubeDescSignatureUpdate {
+ private KylinConfig config = null;
+ private ResourceStore store;
+ private String[] cubeNames;
+ private List<String> updatedResources = Lists.newArrayList();
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
+
+ public CubeDescSignatureUpdate(String[] cubes) {
+ config = KylinConfig.getInstanceFromEnv();
+ store = ResourceStore.getStore(config);
+ cubeNames = cubes;
+ }
+
+ public void update() {
+ logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ List<CubeDesc> cubeDescs;
+ if (ArrayUtils.isEmpty(cubeNames)) {
+ cubeDescs = cubeDescManager.listAllDesc();
+ } else {
+ String[] names = cubeNames[0].split(",");
+ if (ArrayUtils.isEmpty(names))
+ return;
+ cubeDescs = Lists.newArrayListWithCapacity(names.length);
+ for (String name : names) {
+ cubeDescs.add(cubeDescManager.getCubeDesc(name));
+ }
+ }
+ for (CubeDesc cubeDesc : cubeDescs) {
+ updateCubeDesc(cubeDesc);
+ }
+ }
+
+ public static void main(String args[]) {
+ if (args != null && args.length != 0 && args.length != 1) {
+ System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 ");
+ return;
+ }
+
+ CubeDescSignatureUpdate metadataUpgrade = new CubeDescSignatureUpdate(args);
+ metadataUpgrade.update();
+
+ logger.info("=================================================================");
+ logger.info("Run CubeDescSignatureUpdate completed;");
+
+ if (!metadataUpgrade.updatedResources.isEmpty()) {
+ logger.info("Following resources are updated successfully:");
+ for (String s : metadataUpgrade.updatedResources) {
+ logger.info(s);
+ }
+ } else {
+ logger.warn("No resource updated.");
+ }
+
+ if (!metadataUpgrade.errorMsgs.isEmpty()) {
+ logger.info("Here are the error/warning messages, you may need to check:");
+ for (String s : metadataUpgrade.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The update succeeds.");
+ }
+
+ logger.info("=================================================================");
+ }
+
+ private void updateCubeDesc(CubeDesc cubeDesc) {
+ try {
+ String calculatedSign = cubeDesc.calculateSignature();
+ if (!cubeDesc.getSignature().equals(calculatedSign))
+ {
+ cubeDesc.setSignature(calculatedSign);
+ store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+ updatedResources.add(cubeDesc.getResourcePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/976d88bc/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 15dc993..a5ae09c 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -282,10 +282,9 @@ public class DeployCoprocessorCLI {
}
String jarPath = valueMatcher.group(1).trim();
- //String clsName = valueMatcher.group(2).trim();
- //if (CubeObserverClass.equals(clsName)) {
- result.add(jarPath);
- //}
+ if (StringUtils.isNotEmpty(jarPath)) {
+ result.add(jarPath);
+ }
}
}
[10/10] kylin git commit: KYLIN-919 model name validate when edit
model, add lowercase check
Posted by ma...@apache.org.
KYLIN-919 model name validate when edit model,add lowercase check
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/83288ac5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/83288ac5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/83288ac5
Branch: refs/heads/2.0-rc
Commit: 83288ac56c1a60f94ecf1646059fb15353471136
Parents: d03c88b
Author: Zhong <ji...@lm-shc-16501192.corp.ebay.com>
Authored: Wed Dec 2 15:02:03 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:26:30 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/modelSchema.js | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/83288ac5/webapp/app/js/controllers/modelSchema.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelSchema.js b/webapp/app/js/controllers/modelSchema.js
index 9e9f09c..682ce5e 100644
--- a/webapp/app/js/controllers/modelSchema.js
+++ b/webapp/app/js/controllers/modelSchema.js
@@ -166,9 +166,11 @@ KylinApp.controller('ModelSchemaCtrl', function ($scope, QueryService, UserServi
var modelName = $scope.modelsManager.selectedModel.name.toUpperCase();
var models = $scope.modelsManager.modelNameList;
- if ($scope.modelMode=="addNewModel"&&models.indexOf(modelName) != -1 || models.indexOf(modelName.toLowerCase()) !=-1) {
- SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 'warning');
- return false;
+ if ($scope.modelMode=="addNewModel") {
+ if(models.indexOf(modelName) != -1 || models.indexOf(modelName.toLowerCase()) !=-1){
+ SweetAlert.swal('', "Model named [" + modelName + "] already exist!", 'warning');
+ return false;
+ }
}
return true;
}
[03/10] kylin git commit: KYLIN-1180 Fix some NPE on Dictionary
Posted by ma...@apache.org.
KYLIN-1180 Fix some NPE on Dictionary
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1314fdaf
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1314fdaf
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1314fdaf
Branch: refs/heads/2.0-rc
Commit: 1314fdafc00f40f6989492ac3eb4d7b1e74bb3a3
Parents: b2c0963
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 30 10:52:08 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:20 2015 +0800
----------------------------------------------------------------------
.../java/org/apache/kylin/dict/TableColumnValueEnumerator.java | 2 +-
.../java/org/apache/kylin/dict/lookup/LookupBytesTable.java | 4 ++++
.../java/org/apache/kylin/dict/lookup/LookupStringTable.java | 3 +++
.../main/java/org/apache/kylin/dict/lookup/LookupTable.java | 6 ++++--
4 files changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1314fdaf/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
index ab9a6ff..5f9460c 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/TableColumnValueEnumerator.java
@@ -53,7 +53,7 @@ public class TableColumnValueEnumerator implements IDictionaryValueEnumerator {
colStrValue = split[colIndex];
}
- colValue = Bytes.toBytes(colStrValue);
+ colValue = colStrValue == null ? null : Bytes.toBytes(colStrValue);
return true;
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/1314fdaf/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
index 081a97e..0758edc 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupBytesTable.java
@@ -49,4 +49,8 @@ public class LookupBytesTable extends LookupTable<ByteArray> {
return cell.toString();
}
+ public Class<?> getType() {
+ return ByteArray.class;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1314fdaf/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
index 2d92d68..ce73feb 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupStringTable.java
@@ -43,4 +43,7 @@ public class LookupStringTable extends LookupTable<String> {
return cell;
}
+ public Class<?> getType() {
+ return String.class;
+ }
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/1314fdaf/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
index 5d43542..eb2b963 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/lookup/LookupTable.java
@@ -36,7 +36,7 @@ import com.google.common.collect.Sets;
/**
* An in-memory lookup table, in which each cell is an object of type T. The
* table is indexed by specified PK for fast lookup.
- *
+ *
* @author yangli9
*/
abstract public class LookupTable<T extends Comparable<T>> {
@@ -73,7 +73,7 @@ abstract public class LookupTable<T extends Comparable<T>> {
@SuppressWarnings("unchecked")
private void initRow(String[] cols, int[] keyIndex) {
T[] value = convertRow(cols);
- T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(value[0].getClass(), keyIndex.length);
+ T[] keyCols = (T[]) java.lang.reflect.Array.newInstance(getType(), keyIndex.length);
for (int i = 0; i < keyCols.length; i++)
keyCols[i] = value[keyIndex[i]];
@@ -162,6 +162,8 @@ abstract public class LookupTable<T extends Comparable<T>> {
abstract protected String toString(T cell);
+ abstract public Class<?> getType();
+
public void dump() {
for (Array<T> key : data.keySet()) {
System.out.println(toString(key.data) + " => " + toString(data.get(key)));
[09/10] kylin git commit: KYLIN-919 fix ui show dimension error
Posted by ma...@apache.org.
KYLIN-919 fix ui show dimension error
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d03c88b8
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d03c88b8
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d03c88b8
Branch: refs/heads/2.0-rc
Commit: d03c88b83966a32a32d955cdcb3782615d06f79b
Parents: 3ecbfe7
Author: jian <ji...@163.com>
Authored: Fri Dec 4 12:29:09 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:26:17 2015 +0800
----------------------------------------------------------------------
webapp/app/js/controllers/cubeEdit.js | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d03c88b8/webapp/app/js/controllers/cubeEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/cubeEdit.js b/webapp/app/js/controllers/cubeEdit.js
index 2920458..ae4a0f1 100755
--- a/webapp/app/js/controllers/cubeEdit.js
+++ b/webapp/app/js/controllers/cubeEdit.js
@@ -47,6 +47,7 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
return temp;
};
+ //get columns from model
$scope.getDimColumnsByTable = function (tableName) {
if (!tableName) {
return [];
@@ -55,6 +56,9 @@ KylinApp.controller('CubeEditCtrl', function ($scope, $q, $routeParams, $locatio
var tableDim = _.find($scope.metaModel.model.dimensions, function (dimension) {
return dimension.table == tableName
});
+ if(!tableDim){
+ return [];
+ }
var tableDimColumns = tableDim.columns;
var avaColObject = _.filter(tableColumns, function (col) {
return tableDimColumns.indexOf(col.name) != -1;
[05/10] kylin git commit: KYLIN-1064 Restore disabled queries in
KylinQueryTest.testVerifyQuery
Posted by ma...@apache.org.
KYLIN-1064 Restore disabled queries in KylinQueryTest.testVerifyQuery
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/2fb097bb
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/2fb097bb
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/2fb097bb
Branch: refs/heads/2.0-rc
Commit: 2fb097bba79414b70b8a025b893a9ebf59a11258
Parents: fde4a3d
Author: lidongsjtu <do...@ebay.com>
Authored: Thu Dec 3 09:49:38 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:34 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/query/relnode/OLAPLimitRel.java | 7 +++++--
.../resources/query/sql_verifyCount/query04.sql | 19 +++++++++++++++++++
.../query/sql_verifyCount/query04.sql.disable | 19 -------------------
.../resources/query/sql_verifyCount/query05.sql | 19 +++++++++++++++++++
.../query/sql_verifyCount/query05.sql.disable | 19 -------------------
.../resources/query/sql_verifyCount/query06.sql | 19 +++++++++++++++++++
.../query/sql_verifyCount/query06.sql.disable | 19 -------------------
.../resources/query/sql_verifyCount/query07.sql | 19 +++++++++++++++++++
.../query/sql_verifyCount/query07.sql.disable | 19 -------------------
9 files changed, 81 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
----------------------------------------------------------------------
diff --git a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
index 74d5de0..572a5c7 100644
--- a/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
+++ b/query/src/main/java/org/apache/kylin/query/relnode/OLAPLimitRel.java
@@ -102,8 +102,11 @@ public class OLAPLimitRel extends SingleRel implements OLAPRel {
@Override
public EnumerableRel implementEnumerable(List<EnumerableRel> inputs) {
- return new EnumerableLimit(getCluster(), getCluster().traitSetOf(EnumerableConvention.INSTANCE), //
- sole(inputs), localOffset, localFetch);
+ EnumerableRel input = sole(inputs);
+ if (input instanceof OLAPRel) {
+ ((OLAPRel) input).replaceTraitSet(EnumerableConvention.INSTANCE);
+ }
+ return EnumerableLimit.create(input, localOffset, localFetch);
}
@Override
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql b/query/src/test/resources/query/sql_verifyCount/query04.sql
new file mode 100644
index 0000000..9d3e409
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query04.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select * from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable b/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
deleted file mode 100644
index 9d3e409..0000000
--- a/query/src/test/resources/query/sql_verifyCount/query04.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select * from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query05.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query05.sql b/query/src/test/resources/query/sql_verifyCount/query05.sql
new file mode 100644
index 0000000..753a85a
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query05.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select price from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query05.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query05.sql.disable b/query/src/test/resources/query/sql_verifyCount/query05.sql.disable
deleted file mode 100644
index 753a85a..0000000
--- a/query/src/test/resources/query/sql_verifyCount/query05.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select price from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query06.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query06.sql b/query/src/test/resources/query/sql_verifyCount/query06.sql
new file mode 100644
index 0000000..39578a4
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query06.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select lstg_format_name from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query06.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query06.sql.disable b/query/src/test/resources/query/sql_verifyCount/query06.sql.disable
deleted file mode 100644
index 39578a4..0000000
--- a/query/src/test/resources/query/sql_verifyCount/query06.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select lstg_format_name from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query07.sql
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query07.sql b/query/src/test/resources/query/sql_verifyCount/query07.sql
new file mode 100644
index 0000000..0afb493
--- /dev/null
+++ b/query/src/test/resources/query/sql_verifyCount/query07.sql
@@ -0,0 +1,19 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+select price,lstg_format_name from test_kylin_fact limit 100
http://git-wip-us.apache.org/repos/asf/kylin/blob/2fb097bb/query/src/test/resources/query/sql_verifyCount/query07.sql.disable
----------------------------------------------------------------------
diff --git a/query/src/test/resources/query/sql_verifyCount/query07.sql.disable b/query/src/test/resources/query/sql_verifyCount/query07.sql.disable
deleted file mode 100644
index 0afb493..0000000
--- a/query/src/test/resources/query/sql_verifyCount/query07.sql.disable
+++ /dev/null
@@ -1,19 +0,0 @@
---
--- Licensed to the Apache Software Foundation (ASF) under one
--- or more contributor license agreements. See the NOTICE file
--- distributed with this work for additional information
--- regarding copyright ownership. The ASF licenses this file
--- to you under the Apache License, Version 2.0 (the
--- "License"); you may not use this file except in compliance
--- with the License. You may obtain a copy of the License at
---
--- http://www.apache.org/licenses/LICENSE-2.0
---
--- Unless required by applicable law or agreed to in writing, software
--- distributed under the License is distributed on an "AS IS" BASIS,
--- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
--- See the License for the specific language governing permissions and
--- limitations under the License.
---
-
-select price,lstg_format_name from test_kylin_fact limit 100
[06/10] kylin git commit: KYLIN-1190 Make memory budget per query
configurable
Posted by ma...@apache.org.
KYLIN-1190 Make memory budget per query configurable
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a5875d03
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a5875d03
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a5875d03
Branch: refs/heads/2.0-rc
Commit: a5875d033a8d9a913f5c76e44c44ebd3c7857150
Parents: 2fb097b
Author: lidongsjtu <do...@ebay.com>
Authored: Thu Dec 3 09:50:19 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:39 2015 +0800
----------------------------------------------------------------------
.../main/java/org/apache/kylin/common/KylinConfig.java | 4 ++++
.../kylin/storage/hbase/cube/v1/CubeStorageQuery.java | 12 ++++++++----
.../kylin/storage/hbase/cube/v2/CubeStorageQuery.java | 11 +++++++----
3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index 9e6c64b..86b6a9f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -495,6 +495,10 @@ public class KylinConfig implements Serializable {
return Long.parseLong(this.getOptional("kylin.query.cache.threshold.scancount", String.valueOf(10 * 1024)));
}
+ public long getQueryMemBudget() {
+ return Long.parseLong(this.getOptional("kylin.query.mem.budget", String.valueOf(3L * 1024 * 1024 * 1024)));
+ }
+
public boolean isQuerySecureEnabled() {
return Boolean.parseBoolean(this.getOptional("kylin.query.security.enabled", "false"));
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
index 4d34943..2fa0490 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeStorageQuery.java
@@ -82,7 +82,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
private static final int MERGE_KEYRANGE_THRESHOLD = 100;
- private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
@@ -732,8 +731,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
}
}
- long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
- context.setThreshold((int) rowEst);
+ long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst;
+ if (rowEst > 0) {
+ logger.info("Memory budget is set to: " + rowEst);
+ context.setThreshold((int) rowEst);
+ } else {
+ logger.info("Memory budget is not set.");
+ }
}
private void setLimit(TupleFilter filter, StorageContext context) {
@@ -760,7 +764,7 @@ public class CubeStorageQuery implements ICachableStorageQuery {
topnLiteralCol = func.getTopNLiteralColumn();
}
}
-
+
// if TopN is not involved
if (topnFunc == null)
return;
http://git-wip-us.apache.org/repos/asf/kylin/blob/a5875d03/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
index 0c8c3bd..59ed99b 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeStorageQuery.java
@@ -41,8 +41,6 @@ public class CubeStorageQuery implements ICachableStorageQuery {
private static final Logger logger = LoggerFactory.getLogger(CubeStorageQuery.class);
- private static final long MEM_BUDGET_PER_QUERY = 3L * 1024 * 1024 * 1024; // 3G
-
private final CubeInstance cubeInstance;
private final CubeDesc cubeDesc;
@@ -362,8 +360,13 @@ public class CubeStorageQuery implements ICachableStorageQuery {
rowSizeEst += func.getReturnDataType().getStorageBytesEstimate();
}
- long rowEst = MEM_BUDGET_PER_QUERY / rowSizeEst;
- context.setThreshold((int) rowEst);
+ long rowEst = this.cubeInstance.getConfig().getQueryMemBudget() / rowSizeEst;
+ if (rowEst > 0) {
+ logger.info("Memory budget is set to: " + rowEst);
+ context.setThreshold((int) rowEst);
+ } else {
+ logger.info("Memory budget is not set.");
+ }
}
private void setLimit(TupleFilter filter, StorageContext context) {
[08/10] kylin git commit: KYLIN-1182 Upgrade tool for DataModelDesc
from 1.x to 2.0
Posted by ma...@apache.org.
KYLIN-1182 Upgrade tool for DataModelDesc from 1.x to 2.0
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3ecbfe72
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3ecbfe72
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3ecbfe72
Branch: refs/heads/2.0-rc
Commit: 3ecbfe72d94547d9cd3d4c90c98024c8fc915ce5
Parents: 1b6f1fe
Author: lidongsjtu <do...@ebay.com>
Authored: Thu Dec 3 11:44:59 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:52 2015 +0800
----------------------------------------------------------------------
.../job/upgrade/CubeDescSignatureUpdate.java | 66 ++++---
.../job/upgrade/v2/DataModelDescUpgradeV2.java | 170 +++++++++++++++++++
.../kylin/metadata/model/DataModelDesc.java | 8 +
3 files changed, 217 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ecbfe72/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
index 178f1b0..36784de 100644
--- a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
@@ -18,57 +18,40 @@
package org.apache.kylin.job.upgrade;
-import com.google.common.collect.Lists;
+import java.util.List;
+
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
-import java.util.List;
+import com.google.common.collect.Lists;
/**
* Created by dongli on 11/17/15.
*/
public class CubeDescSignatureUpdate {
+ private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
private KylinConfig config = null;
private ResourceStore store;
private String[] cubeNames;
private List<String> updatedResources = Lists.newArrayList();
private List<String> errorMsgs = Lists.newArrayList();
- private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
-
public CubeDescSignatureUpdate(String[] cubes) {
config = KylinConfig.getInstanceFromEnv();
store = ResourceStore.getStore(config);
cubeNames = cubes;
}
- public void update() {
- logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
- CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
- List<CubeDesc> cubeDescs;
- if (ArrayUtils.isEmpty(cubeNames)) {
- cubeDescs = cubeDescManager.listAllDesc();
- } else {
- String[] names = cubeNames[0].split(",");
- if (ArrayUtils.isEmpty(names))
- return;
- cubeDescs = Lists.newArrayListWithCapacity(names.length);
- for (String name : names) {
- cubeDescs.add(cubeDescManager.getCubeDesc(name));
- }
- }
- for (CubeDesc cubeDesc : cubeDescs) {
- updateCubeDesc(cubeDesc);
- }
- }
-
public static void main(String args[]) {
- if (args != null && args.length != 0 && args.length != 1) {
+ if (args != null && args.length > 1) {
System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 ");
return;
}
@@ -100,11 +83,40 @@ public class CubeDescSignatureUpdate {
logger.info("=================================================================");
}
+ public void update() {
+ logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ List<CubeDesc> cubeDescs;
+ if (ArrayUtils.isEmpty(cubeNames)) {
+ cubeDescs = cubeDescManager.listAllDesc();
+ } else {
+ String[] names = cubeNames[0].split(",");
+ if (ArrayUtils.isEmpty(names))
+ return;
+ cubeDescs = Lists.newArrayListWithCapacity(names.length);
+ for (String name : names) {
+ cubeDescs.add(cubeDescManager.getCubeDesc(name));
+ }
+ }
+ for (CubeDesc cubeDesc : cubeDescs) {
+ updateCubeDesc(cubeDesc);
+ }
+
+ verify();
+ }
+
+ private void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
+ }
+
private void updateCubeDesc(CubeDesc cubeDesc) {
try {
String calculatedSign = cubeDesc.calculateSignature();
- if (!cubeDesc.getSignature().equals(calculatedSign))
- {
+ if (!cubeDesc.getSignature().equals(calculatedSign)) {
cubeDesc.setSignature(calculatedSign);
store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
updatedResources.add(cubeDesc.getResourcePath());
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ecbfe72/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
new file mode 100644
index 0000000..81b277d
--- /dev/null
+++ b/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.job.upgrade.v2;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DimensionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * Created by dongli on 11/30/15.
+ *
+ * In v1.x, DataModelDesc doesn't include Dimensions and Measures, this tool is to fill them from CubeDesc.
+ */
+public class DataModelDescUpgradeV2 {
+ private static final Log logger = LogFactory.getLog(DataModelDescUpgradeV2.class);
+ private KylinConfig config = null;
+ private ResourceStore store;
+ private String[] models;
+ private List<String> updatedResources = Lists.newArrayList();
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ public DataModelDescUpgradeV2(String[] models) {
+ this.config = KylinConfig.getInstanceFromEnv();
+ this.store = ResourceStore.getStore(config);
+ this.models = models;
+ }
+
+ public static void main(String args[]) throws Exception {
+ if (args != null && args.length > 1) {
+ System.out.println("Usage: java DataModelDescUpradeV2 [Models]; e.g, model1,model2 ");
+ return;
+ }
+
+ DataModelDescUpgradeV2 metadataUpgrade = new DataModelDescUpgradeV2(args);
+ metadataUpgrade.upgrade();
+
+ logger.info("=================================================================");
+ logger.info("Run DataModelDescUpradeV2 completed;");
+
+ if (!metadataUpgrade.updatedResources.isEmpty()) {
+ logger.info("Following resources are updated successfully:");
+ for (String s : metadataUpgrade.updatedResources) {
+ logger.info(s);
+ }
+ } else {
+ logger.warn("No resource updated.");
+ }
+
+ if (!metadataUpgrade.errorMsgs.isEmpty()) {
+ logger.info("Here are the error/warning messages, you may need to check:");
+ for (String s : metadataUpgrade.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The update succeeds.");
+ }
+
+ logger.info("=================================================================");
+ }
+
+ public void upgrade() {
+ logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc();
+ for (CubeDesc cubeDesc : cubeDescs) {
+ if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, cubeDesc.getModelName())) {
+ upgradeDataModelDesc(cubeDesc);
+ }
+ }
+
+ verify();
+ }
+
+ private void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
+ }
+
+ private void upgradeDataModelDesc(CubeDesc cubeDesc) {
+ boolean upgrade = false;
+ DataModelDesc modelDesc = cubeDesc.getModel();
+ try {
+ if (modelDesc != null && modelDesc.getDimensions() == null && modelDesc.getMetrics() == null) {
+ List<org.apache.kylin.cube.model.DimensionDesc> cubeDimDescList = cubeDesc.getDimensions();
+ if (!CollectionUtils.isEmpty(cubeDimDescList)) {
+ Map<String, HashSet<String>> modelDimMap = Maps.newHashMap();
+ for (org.apache.kylin.cube.model.DimensionDesc cubeDimDesc : cubeDimDescList) {
+ if (!modelDimMap.containsKey(cubeDimDesc.getTable())) {
+ modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>());
+ }
+ modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn()));
+ }
+
+ List<DimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size());
+ for (Map.Entry<String, HashSet<String>> modelDimEntry : modelDimMap.entrySet()) {
+ DimensionDesc dimDesc = new DimensionDesc();
+ dimDesc.setTable(modelDimEntry.getKey());
+ String[] columns = new String[modelDimEntry.getValue().size()];
+ columns = modelDimEntry.getValue().toArray(columns);
+ dimDesc.setColumns(columns);
+ modelDimDescList.add(dimDesc);
+ }
+ DimensionDesc.capicalizeStrings(modelDimDescList);
+ modelDesc.setDimensions(modelDimDescList);
+ upgrade = true;
+ }
+
+ List<MeasureDesc> cubeMeasDescList = cubeDesc.getMeasures();
+ if (!CollectionUtils.isEmpty(cubeDimDescList)) {
+ ArrayList<String> metrics = Lists.newArrayListWithExpectedSize(cubeMeasDescList.size());
+ for (MeasureDesc cubeMeasDesc : cubeMeasDescList) {
+ for (TblColRef tblColRef : cubeMeasDesc.getFunction().getParameter().getColRefs()) {
+ metrics.add(tblColRef.getName());
+ }
+ }
+ String[] metricsArray = new String[metrics.size()];
+ modelDesc.setMetrics(metrics.toArray(metricsArray));
+ upgrade = true;
+ }
+ }
+
+ if (upgrade) {
+ store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER);
+ updatedResources.add(modelDesc.getResourcePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ errorMsgs.add("Update DataModelDesc[" + modelDesc.getName() + "] failed: " + e.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3ecbfe72/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
----------------------------------------------------------------------
diff --git a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
index 66581c0..db53bc5 100644
--- a/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
+++ b/core-metadata/src/main/java/org/apache/kylin/metadata/model/DataModelDesc.java
@@ -322,4 +322,12 @@ public class DataModelDesc extends RootPersistentEntity {
public String[] getMetrics() {
return metrics;
}
+
+ public void setDimensions(List<DimensionDesc> dimensions) {
+ this.dimensions = dimensions;
+ }
+
+ public void setMetrics(String[] metrics) {
+ this.metrics = metrics;
+ }
}
[07/10] kylin git commit: minor,
update hyperlinks for kylin contacts on webapp
Posted by ma...@apache.org.
minor, update hyperlinks for kylin contacts on webapp
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/1b6f1fe6
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/1b6f1fe6
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/1b6f1fe6
Branch: refs/heads/2.0-rc
Commit: 1b6f1fe6280533241e4f8b63b8b7bbca33897296
Parents: a5875d0
Author: lidongsjtu <do...@ebay.com>
Authored: Thu Dec 3 11:38:11 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:45 2015 +0800
----------------------------------------------------------------------
webapp/app/partials/footer.html | 4 ++--
webapp/app/partials/login.html | 2 +-
webapp/app/partials/models/models_tree.html | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b6f1fe6/webapp/app/partials/footer.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/footer.html b/webapp/app/partials/footer.html
index 5a72a52..d0a513a 100644
--- a/webapp/app/partials/footer.html
+++ b/webapp/app/partials/footer.html
@@ -20,8 +20,8 @@
<div class="container">
<div class="copyright">
<span>
- <a href="http://kylin.io" style="color:#808080;"><i class="fa fa-home"></i> Home Page</a> |
- <a href="https://groups.google.com/forum/#!forum/kylin-olap" style="color:#808080;"><i class="fa fa-users"></i> Google Group</a>
+ <a href="http://kylin.apache.org" style="color:#808080;"><i class="fa fa-home"></i> Apache Kylin</a> |
+ <a href="http://kylin.apache.org/community/" style="color:#808080;"><i class="fa fa-users"></i> Apache Kylin Community</a>
</span>
</div>
</div>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b6f1fe6/webapp/app/partials/login.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/login.html b/webapp/app/partials/login.html
index e64f9d5..79128b1 100644
--- a/webapp/app/partials/login.html
+++ b/webapp/app/partials/login.html
@@ -56,7 +56,7 @@
</div>
<div class="space"></div>
<div class="form-group">
- <a href="https://groups.google.com/forum/#!forum/kylin-olap" target="_blank" class="text-muted">Login Issue?</a>
+ <a href="http://kylin.apache.org/community/" target="_blank" class="text-muted">Login Issue?</a>
<button class="pull-right btn btn-sm btn-primary" type="submit" ng-disabled="login_form.$invalid">
<i class="ace-icon fa fa-key"></i>
<span class="bigger-110">{{loading? 'Logging In...': 'Log In'}}</span>
http://git-wip-us.apache.org/repos/asf/kylin/blob/1b6f1fe6/webapp/app/partials/models/models_tree.html
----------------------------------------------------------------------
diff --git a/webapp/app/partials/models/models_tree.html b/webapp/app/partials/models/models_tree.html
index e064792..4e8887b 100644
--- a/webapp/app/partials/models/models_tree.html
+++ b/webapp/app/partials/models/models_tree.html
@@ -38,7 +38,7 @@
</a>
<ul class="dropdown-menu">
<li ng-if="userService.hasRole('ROLE_ADMIN')">
- <a href="models/add" ng-if="userService.hasRole('ROLE_MODELER')"><i class="fa fa-star"></i> New Model</a>
+ <a href="models/add" ng-if="userService.hasRole('ROLE_MODELER')"><i class="fa fa-star"></i>New Model</a>
</li>
<li ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(modelsManager.selectedModel, permissions.ADMINISTRATION.mask, permissions.MANAGEMENT.mask, permissions.OPERATION.mask)">
<a href="cubes/add" ng-if="userService.hasRole('ROLE_ADMIN')"><i class="fa fa-cube"></i>New Cube</a>
[04/10] kylin git commit: KYLIN-1181 Remove limitation of
mapreduce.job.split.metainfo.maxsize for hive jobs
Posted by ma...@apache.org.
KYLIN-1181 Remove limitation of mapreduce.job.split.metainfo.maxsize for hive jobs
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/fde4a3d5
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/fde4a3d5
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/fde4a3d5
Branch: refs/heads/2.0-rc
Commit: fde4a3d511adbbe7b4fec7c3fcaeda4ac9ffda4b
Parents: 1314fda
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 30 10:56:02 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:25 2015 +0800
----------------------------------------------------------------------
build/conf/kylin_hive_conf.xml | 9 +++++++++
1 file changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/fde4a3d5/build/conf/kylin_hive_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_hive_conf.xml b/build/conf/kylin_hive_conf.xml
index b77b664..afa53f7 100644
--- a/build/conf/kylin_hive_conf.xml
+++ b/build/conf/kylin_hive_conf.xml
@@ -67,4 +67,13 @@
<value>false</value>
<description>Hive concurrency lock</description>
</property>
+
+ <property>
+ <name>mapreduce.job.split.metainfo.maxsize</name>
+ <value>-1</value>
+ <description>The maximum permissible size of the split metainfo file.
+ The JobTracker won't attempt to read split metainfo files bigger than
+ the configured value. No limits if set to -1.
+ </description>
+ </property>
</configuration>
\ No newline at end of file
[02/10] kylin git commit: KYLIN-1179 Cannot use String as partition
column
Posted by ma...@apache.org.
KYLIN-1179 Cannot use String as partition column
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/b2c0963b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/b2c0963b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/b2c0963b
Branch: refs/heads/2.0-rc
Commit: b2c0963bc936d86b60e87cff578491a5eb9bfbed
Parents: 976d88b
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Nov 30 10:48:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Fri Dec 4 16:14:14 2015 +0800
----------------------------------------------------------------------
.../hbase/cube/v2/CubeSegmentScanner.java | 26 +++++++++++---------
webapp/app/js/controllers/modelEdit.js | 2 +-
2 files changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2c0963b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
index 1eed318..2f78a86 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v2/CubeSegmentScanner.java
@@ -11,6 +11,7 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Set;
+import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.debug.BackdoorToggles;
import org.apache.kylin.common.util.ByteArray;
import org.apache.kylin.common.util.DateFormat;
@@ -30,6 +31,7 @@ import org.apache.kylin.gridtable.GTScanRequest;
import org.apache.kylin.gridtable.GTUtil;
import org.apache.kylin.gridtable.IGTScanner;
import org.apache.kylin.metadata.filter.TupleFilter;
+import org.apache.kylin.metadata.model.DataType;
import org.apache.kylin.metadata.model.FunctionDesc;
import org.apache.kylin.metadata.model.TblColRef;
@@ -83,27 +85,23 @@ public class CubeSegmentScanner implements IGTScanner {
GTInfo trimmedInfo = GTInfo.deserialize(trimmedInfoBytes);
for (GTScanRange range : scanRanges) {
- scanRequests.add(new GTScanRequest(trimmedInfo, range.replaceGTInfo(trimmedInfo),
- gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate));
+ scanRequests.add(new GTScanRequest(trimmedInfo, range.replaceGTInfo(trimmedInfo), gtDimensions, gtAggrGroups, gtAggrMetrics, gtAggrFuncs, gtFilter, allowPreAggregate));
}
scanner = new Scanner();
}
private Pair<ByteArray, ByteArray> getSegmentStartAndEnd(TblColRef tblColRef, int index) {
-
- String partitionColType = tblColRef.getColumnDesc().getDatatype();
-
ByteArray start;
if (cubeSeg.getDateRangeStart() != Long.MIN_VALUE) {
- start = translateTsToString(cubeSeg.getDateRangeStart(), partitionColType, index);
+ start = translateTsToString(cubeSeg.getDateRangeStart(), index, 1);
} else {
start = new ByteArray();
}
ByteArray end;
if (cubeSeg.getDateRangeEnd() != Long.MAX_VALUE) {
- end = translateTsToString(cubeSeg.getDateRangeEnd(), partitionColType, index);
+ end = translateTsToString(cubeSeg.getDateRangeEnd(), index, -1);
} else {
end = new ByteArray();
}
@@ -111,19 +109,25 @@ public class CubeSegmentScanner implements IGTScanner {
}
- private ByteArray translateTsToString(long ts, String partitionColType, int index) {
+ private ByteArray translateTsToString(long ts, int index, int roundingFlag) {
String value;
- if ("date".equalsIgnoreCase(partitionColType)) {
+ DataType partitionColType = info.getColumnType(index);
+ if (partitionColType.isDate()) {
value = DateFormat.formatToDateStr(ts);
- } else if ("timestamp".equalsIgnoreCase(partitionColType)) {
+ } else if (partitionColType.isDatetime()) {
//TODO: if partition col is not dict encoded, value's format may differ from expected. Though by default it is not the case
value = DateFormat.formatToTimeWithoutMilliStr(ts);
+ } else if (partitionColType.isStringFamily()) {
+ String partitionDateFormat = cubeSeg.getCubeDesc().getModel().getPartitionDesc().getPartitionDateFormat();
+ if (StringUtils.isEmpty(partitionDateFormat))
+ partitionDateFormat = DateFormat.DEFAULT_DATE_PATTERN;
+ value = DateFormat.formatToDateStr(ts, partitionDateFormat);
} else {
throw new RuntimeException("Type " + partitionColType + " is not valid partition column type");
}
ByteBuffer buffer = ByteBuffer.allocate(info.getMaxColumnLength());
- info.getCodeSystem().encodeColumnValue(index, value, buffer);
+ info.getCodeSystem().encodeColumnValue(index, value, roundingFlag, buffer);
return ByteArray.copyOf(buffer.array(), 0, buffer.position());
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/b2c0963b/webapp/app/js/controllers/modelEdit.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/controllers/modelEdit.js b/webapp/app/js/controllers/modelEdit.js
index cbfb57f..70d4159 100644
--- a/webapp/app/js/controllers/modelEdit.js
+++ b/webapp/app/js/controllers/modelEdit.js
@@ -35,7 +35,7 @@ KylinApp.controller('ModelEditCtrl', function ($scope, $q, $routeParams, $locati
$scope.getPartitonColumns = function(tableName){
var columns = _.filter($scope.getColumnsByTable(tableName),function(column){
- return column.datatype==="date"||column.datatype==="string"||column.datatype==="timestamp";
+ return column.datatype==="date"||column.datatype==="timestamp"||column.datatype==="string"||column.datatype.startsWith("varchar");
});
return columns;
};