You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2015/12/14 03:30:56 UTC
[1/2] kylin git commit: minor,
update storage_type and engine_type in upgrade tool
Repository: kylin
Updated Branches:
refs/heads/2.0-rc e8de0dc50 -> d4bf5c28e
minor, update storage_type and engine_type in upgrade tool
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/3722d383
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/3722d383
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/3722d383
Branch: refs/heads/2.0-rc
Commit: 3722d383ce98b04ce30df9c183e2e9e8b293d88b
Parents: e8de0dc
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Dec 14 09:11:17 2015 +0800
Committer: lidongsjtu <do...@ebay.com>
Committed: Mon Dec 14 09:11:33 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/cube/model/CubeDesc.java | 4 +--
.../cube/upgrade/v2/CubeMetadataUpgradeV2.java | 30 +++++++++++++++++++-
2 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/3722d383/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
index 17dc650..e78dea5 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/model/CubeDesc.java
@@ -804,7 +804,7 @@ public class CubeDesc extends RootPersistentEntity {
return storageType;
}
- void setStorageType(int storageType) {
+ public void setStorageType(int storageType) {
this.storageType = storageType;
}
@@ -812,7 +812,7 @@ public class CubeDesc extends RootPersistentEntity {
return engineType;
}
- void setEngineType(int engineType) {
+ public void setEngineType(int engineType) {
this.engineType = engineType;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/3722d383/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
index 3b6ba9d..77cb28d 100644
--- a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
@@ -30,12 +30,15 @@ import org.apache.commons.logging.LogFactory;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeInstance;
import org.apache.kylin.cube.CubeManager;
import org.apache.kylin.cube.model.CubeDesc;
import org.apache.kylin.cube.upgrade.CubeDescSignatureUpdate;
import org.apache.kylin.metadata.MetadataManager;
import org.apache.kylin.metadata.model.DataModelDesc;
import org.apache.kylin.metadata.model.DimensionDesc;
+import org.apache.kylin.metadata.model.IEngineAware;
+import org.apache.kylin.metadata.model.IStorageAware;
import org.apache.kylin.metadata.model.MeasureDesc;
import org.apache.kylin.metadata.model.TblColRef;
import org.apache.kylin.metadata.project.ProjectManager;
@@ -73,6 +76,7 @@ public class CubeMetadataUpgradeV2 {
CubeMetadataUpgradeV2 metadataUpgrade = new CubeMetadataUpgradeV2(args);
metadataUpgrade.upgradeModelMetadata();
+ metadataUpgrade.upgradeEmptyCubes();
metadataUpgrade.verify();
logger.info("=================================================================");
@@ -119,8 +123,9 @@ public class CubeMetadataUpgradeV2 {
ProjectManager.getInstance(config);
}
+ // Recalculate signature for CubeDesc
private void upgradeCubeDescSignature(CubeDesc cubeDesc) {
- CubeDescSignatureUpdate cubeDescSignatureUpdate = new CubeDescSignatureUpdate(new String[] {cubeDesc.getName()});
+ CubeDescSignatureUpdate cubeDescSignatureUpdate = new CubeDescSignatureUpdate(new String[] { cubeDesc.getName() });
cubeDescSignatureUpdate.update();
if (CollectionUtils.isEmpty(cubeDescSignatureUpdate.getErrorMsgs())) {
@@ -131,6 +136,29 @@ public class CubeMetadataUpgradeV2 {
}
}
+ // Update engine_type and storage_type to 2.0, if the cube has no segments.
+ private void upgradeEmptyCubes() {
+ CubeManager cubeManager = CubeManager.getInstance(config);
+ List<CubeInstance> cubes = cubeManager.listAllCubes();
+ for (CubeInstance cube : cubes) {
+ try {
+ CubeDesc cubeDesc = cube.getDescriptor();
+ if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, cubeDesc.getModelName())) {
+ if (cube.getFirstSegment() == null && cubeDesc != null && cubeDesc.getStorageType() == IStorageAware.ID_HBASE && cubeDesc.getEngineType() == IEngineAware.ID_MR_V1) {
+ cubeDesc.setEngineType(IEngineAware.ID_MR_V2);
+ cubeDesc.setStorageType(IStorageAware.ID_SHARDED_HBASE);
+
+ store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+ updatedResources.add(cubeDesc.getResourcePath());
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ errorMsgs.add("Update Cube[" + cube.getName() + "] failed: " + e.getLocalizedMessage());
+ }
+ }
+ }
+
private void upgradeDataModelDesc(CubeDesc cubeDesc) {
boolean upgrade = false;
DataModelDesc modelDesc = cubeDesc.getModel();
[2/2] kylin git commit: minor,
set default memory parameters for cubing jobs
Posted by li...@apache.org.
minor, set default memory parameters for cubing jobs
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d4bf5c28
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d4bf5c28
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d4bf5c28
Branch: refs/heads/2.0-rc
Commit: d4bf5c28e0db2dcc292b5b4f9f8b8db6350bb01f
Parents: 3722d38
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Dec 14 10:30:36 2015 +0800
Committer: lidongsjtu <do...@ebay.com>
Committed: Mon Dec 14 10:30:36 2015 +0800
----------------------------------------------------------------------
build/conf/kylin_job_conf.xml | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d4bf5c28/build/conf/kylin_job_conf.xml
----------------------------------------------------------------------
diff --git a/build/conf/kylin_job_conf.xml b/build/conf/kylin_job_conf.xml
index 15ea683..d150b5b 100644
--- a/build/conf/kylin_job_conf.xml
+++ b/build/conf/kylin_job_conf.xml
@@ -55,8 +55,6 @@
<description>Block replication</description>
</property>
- <!--Properties for calculating cube by splits (in-mem), with which each Mapper need more mem to hold a full cube segment -->
- <!--
<property>
<name>mapreduce.map.java.opts</name>
<value>-Xmx3072m</value>
@@ -65,7 +63,7 @@
<name>mapreduce.map.memory.mb</name>
<value>4096</value>
</property>
- -->
+
<property>
<name>mapred.task.timeout</name>
<value>3600000</value>