You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/12/07 04:35:26 UTC
kylin git commit: KYLIN-1182 Add upgrading script
Repository: kylin
Updated Branches:
refs/heads/2.0-rc 519b43aa9 -> 35052ed37
KYLIN-1182 Add upgrading script
Signed-off-by: honma <ho...@ebay.com>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/35052ed3
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/35052ed3
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/35052ed3
Branch: refs/heads/2.0-rc
Commit: 35052ed374a114021f17b3eda09cddce4d65f8ef
Parents: 519b43a
Author: lidongsjtu <do...@ebay.com>
Authored: Mon Dec 7 09:17:18 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Dec 7 11:41:38 2015 +0800
----------------------------------------------------------------------
build/script/build.sh | 17 ++
build/script/compress.sh | 17 ++
build/script/download-tomcat.sh | 17 ++
build/script/package.sh | 17 ++
build/script/prepare.sh | 17 ++
build/script/prepare_libs.sh | 17 ++
build/script/upgrade_v2.sh | 29 +++
.../cube/upgrade/CubeDescSignatureUpdate.java | 134 +++++++++++++
.../cube/upgrade/v2/CubeMetadataUpgradeV2.java | 186 +++++++++++++++++++
.../job/upgrade/CubeDescSignatureUpdate.java | 129 -------------
.../job/upgrade/v2/DataModelDescUpgradeV2.java | 170 -----------------
11 files changed, 451 insertions(+), 299 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/build.sh
----------------------------------------------------------------------
diff --git a/build/script/build.sh b/build/script/build.sh
index 558d720..f54d41c 100755
--- a/build/script/build.sh
+++ b/build/script/build.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
echo "package back-end"
dir=$(dirname ${0})
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/compress.sh
----------------------------------------------------------------------
diff --git a/build/script/compress.sh b/build/script/compress.sh
index cfd3c18..503fdf1 100755
--- a/build/script/compress.sh
+++ b/build/script/compress.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
# pack webapp into kylin.war so that we have a all-in-one war
dir=$(dirname ${0})
cd ${dir}/../..
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/download-tomcat.sh
----------------------------------------------------------------------
diff --git a/build/script/download-tomcat.sh b/build/script/download-tomcat.sh
index ab649dc..e11aa6f 100755
--- a/build/script/download-tomcat.sh
+++ b/build/script/download-tomcat.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
dir=$(dirname ${0})
cd ${dir}/../..
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/package.sh
----------------------------------------------------------------------
diff --git a/build/script/package.sh b/build/script/package.sh
index 75d810c..a505501 100755
--- a/build/script/package.sh
+++ b/build/script/package.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
echo "Checking maven..."
if [ -z "$(command -v mvn)" ]
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/prepare.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare.sh b/build/script/prepare.sh
index 1b208c4..a2019b8 100755
--- a/build/script/prepare.sh
+++ b/build/script/prepare.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
dir=$(dirname ${0})
cd ${dir}/../..
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/prepare_libs.sh
----------------------------------------------------------------------
diff --git a/build/script/prepare_libs.sh b/build/script/prepare_libs.sh
index 57888ff..ddcfb86 100755
--- a/build/script/prepare_libs.sh
+++ b/build/script/prepare_libs.sh
@@ -1,5 +1,22 @@
#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
dir=$(dirname ${0})
cd ${dir}/../..
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/build/script/upgrade_v2.sh
----------------------------------------------------------------------
diff --git a/build/script/upgrade_v2.sh b/build/script/upgrade_v2.sh
new file mode 100755
index 0000000..a954d35
--- /dev/null
+++ b/build/script/upgrade_v2.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dir=$(dirname ${0})
+source ${dir}/check-env.sh
+
+echo "======Deploy coprocessor======="
+$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-incubating-SNAPSHOT.jar
+
+echo "=====Upgrade Cube metadata====="
+$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.v2.CubeMetadataUpgradeV2
+
+echo "==============End=============="
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
new file mode 100644
index 0000000..b482710
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/CubeDescSignatureUpdate.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.kylin.cube.upgrade;
+
+import java.util.List;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Created by dongli on 11/17/15.
+ */
+public class CubeDescSignatureUpdate {
+ private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
+ private KylinConfig config = null;
+ private ResourceStore store;
+ private String[] cubeNames;
+ private List<String> updatedResources = Lists.newArrayList();
+
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ public CubeDescSignatureUpdate(String[] cubes) {
+ config = KylinConfig.getInstanceFromEnv();
+ store = ResourceStore.getStore(config);
+ cubeNames = cubes;
+ }
+
+ public static void main(String args[]) {
+ if (args != null && args.length > 1) {
+ System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 ");
+ return;
+ }
+
+ CubeDescSignatureUpdate metadataUpgrade = new CubeDescSignatureUpdate(args);
+ metadataUpgrade.update();
+
+ logger.info("=================================================================");
+ logger.info("Run CubeDescSignatureUpdate completed;");
+
+ if (!metadataUpgrade.updatedResources.isEmpty()) {
+ logger.info("Following resources are updated successfully:");
+ for (String s : metadataUpgrade.updatedResources) {
+ logger.info(s);
+ }
+ } else {
+ logger.warn("No resource updated.");
+ }
+
+ if (!metadataUpgrade.errorMsgs.isEmpty()) {
+ logger.info("Here are the error/warning messages, you may need to check:");
+ for (String s : metadataUpgrade.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The update succeeds.");
+ }
+
+ logger.info("=================================================================");
+ }
+
+ public List<String> getErrorMsgs() {
+ return errorMsgs;
+ }
+
+ public void update() {
+ logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ List<CubeDesc> cubeDescs;
+ if (ArrayUtils.isEmpty(cubeNames)) {
+ cubeDescs = cubeDescManager.listAllDesc();
+ } else {
+ String[] names = cubeNames[0].split(",");
+ if (ArrayUtils.isEmpty(names))
+ return;
+ cubeDescs = Lists.newArrayListWithCapacity(names.length);
+ for (String name : names) {
+ cubeDescs.add(cubeDescManager.getCubeDesc(name));
+ }
+ }
+ for (CubeDesc cubeDesc : cubeDescs) {
+ updateCubeDesc(cubeDesc);
+ }
+
+ verify();
+ }
+
+ private void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
+ }
+
+ private void updateCubeDesc(CubeDesc cubeDesc) {
+ try {
+ String calculatedSign = cubeDesc.calculateSignature();
+ if (!cubeDesc.getSignature().equals(calculatedSign)) {
+ cubeDesc.setSignature(calculatedSign);
+ store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
+ updatedResources.add(cubeDesc.getResourcePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
new file mode 100644
index 0000000..3b6ba9d
--- /dev/null
+++ b/core-cube/src/main/java/org/apache/kylin/cube/upgrade/v2/CubeMetadataUpgradeV2.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.cube.upgrade.v2;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.persistence.ResourceStore;
+import org.apache.kylin.cube.CubeDescManager;
+import org.apache.kylin.cube.CubeManager;
+import org.apache.kylin.cube.model.CubeDesc;
+import org.apache.kylin.cube.upgrade.CubeDescSignatureUpdate;
+import org.apache.kylin.metadata.MetadataManager;
+import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.DimensionDesc;
+import org.apache.kylin.metadata.model.MeasureDesc;
+import org.apache.kylin.metadata.model.TblColRef;
+import org.apache.kylin.metadata.project.ProjectManager;
+
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+
+/**
+ * back in 1.x branch there was a CubeMetadataUpgrade which is actually CubeMetadataUpgradeV1,
+ * that upgrades metadata store from v1(prior kylin 0.7) to v2.
+ * the major difference is that we split cube desc to cube desc + model desc
+ *
+ * In 2.0 there is a CubeMetadataUpgradeV2 which is responsible for upgrading the metadata store
+ * from 1.x to 2.0. The major actions in that is updating cube desc signature and upgrade model desc.
+ */
+public class CubeMetadataUpgradeV2 {
+ private static final Log logger = LogFactory.getLog(CubeMetadataUpgradeV2.class);
+ private KylinConfig config = null;
+ private ResourceStore store;
+ private String[] models;
+ private List<String> updatedResources = Lists.newArrayList();
+ private List<String> errorMsgs = Lists.newArrayList();
+
+ public CubeMetadataUpgradeV2(String[] models) {
+ this.config = KylinConfig.getInstanceFromEnv();
+ this.store = ResourceStore.getStore(config);
+ this.models = models;
+ }
+
+ public static void main(String args[]) throws Exception {
+ if (args != null && args.length > 1) {
+ System.out.println("Usage: java CubeMetadataUpgradeV2 [Models]; e.g, model1,model2 ");
+ return;
+ }
+
+ CubeMetadataUpgradeV2 metadataUpgrade = new CubeMetadataUpgradeV2(args);
+ metadataUpgrade.upgradeModelMetadata();
+ metadataUpgrade.verify();
+
+ logger.info("=================================================================");
+ logger.info("Run CubeMetadataUpgradeV2 completed;");
+
+ if (!metadataUpgrade.updatedResources.isEmpty()) {
+ logger.info("Following resources are updated successfully:");
+ for (String s : metadataUpgrade.updatedResources) {
+ logger.info(s);
+ }
+ } else {
+ logger.warn("No resource updated.");
+ }
+
+ if (!metadataUpgrade.errorMsgs.isEmpty()) {
+ logger.info("Here are the error/warning messages, you may need to check:");
+ for (String s : metadataUpgrade.errorMsgs) {
+ logger.warn(s);
+ }
+ } else {
+ logger.info("No error or warning messages; The update succeeds.");
+ }
+
+ logger.info("=================================================================");
+ }
+
+ public void upgradeModelMetadata() {
+ logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
+ CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
+ List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc();
+ for (CubeDesc cubeDesc : cubeDescs) {
+ if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, cubeDesc.getModelName())) {
+ upgradeCubeDescSignature(cubeDesc);
+ upgradeDataModelDesc(cubeDesc);
+ }
+ }
+ }
+
+ private void verify() {
+ MetadataManager.getInstance(config).reload();
+ CubeDescManager.clearCache();
+ CubeDescManager.getInstance(config);
+ CubeManager.getInstance(config);
+ ProjectManager.getInstance(config);
+ }
+
+ private void upgradeCubeDescSignature(CubeDesc cubeDesc) {
+ CubeDescSignatureUpdate cubeDescSignatureUpdate = new CubeDescSignatureUpdate(new String[] {cubeDesc.getName()});
+ cubeDescSignatureUpdate.update();
+
+ if (CollectionUtils.isEmpty(cubeDescSignatureUpdate.getErrorMsgs())) {
+ logger.info("CubeDesc Signature updating succeeded: " + cubeDesc.getName());
+ } else {
+ logger.info("CubeDesc Signature updating failed: " + cubeDesc.getName());
+ this.errorMsgs.addAll(cubeDescSignatureUpdate.getErrorMsgs());
+ }
+ }
+
+ private void upgradeDataModelDesc(CubeDesc cubeDesc) {
+ boolean upgrade = false;
+ DataModelDesc modelDesc = cubeDesc.getModel();
+ try {
+ if (modelDesc != null && modelDesc.getDimensions() == null && modelDesc.getMetrics() == null) {
+ List<org.apache.kylin.cube.model.DimensionDesc> cubeDimDescList = cubeDesc.getDimensions();
+ if (!CollectionUtils.isEmpty(cubeDimDescList)) {
+ Map<String, HashSet<String>> modelDimMap = Maps.newHashMap();
+ for (org.apache.kylin.cube.model.DimensionDesc cubeDimDesc : cubeDimDescList) {
+ if (!modelDimMap.containsKey(cubeDimDesc.getTable())) {
+ modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>());
+ }
+ modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn()));
+ }
+
+ List<DimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size());
+ for (Map.Entry<String, HashSet<String>> modelDimEntry : modelDimMap.entrySet()) {
+ DimensionDesc dimDesc = new DimensionDesc();
+ dimDesc.setTable(modelDimEntry.getKey());
+ String[] columns = new String[modelDimEntry.getValue().size()];
+ columns = modelDimEntry.getValue().toArray(columns);
+ dimDesc.setColumns(columns);
+ modelDimDescList.add(dimDesc);
+ }
+ DimensionDesc.capicalizeStrings(modelDimDescList);
+ modelDesc.setDimensions(modelDimDescList);
+ upgrade = true;
+ }
+
+ List<MeasureDesc> cubeMeasDescList = cubeDesc.getMeasures();
+ if (!CollectionUtils.isEmpty(cubeDimDescList)) {
+ ArrayList<String> metrics = Lists.newArrayListWithExpectedSize(cubeMeasDescList.size());
+ for (MeasureDesc cubeMeasDesc : cubeMeasDescList) {
+ for (TblColRef tblColRef : cubeMeasDesc.getFunction().getParameter().getColRefs()) {
+ metrics.add(tblColRef.getName());
+ }
+ }
+ String[] metricsArray = new String[metrics.size()];
+ modelDesc.setMetrics(metrics.toArray(metricsArray));
+ upgrade = true;
+ }
+ }
+
+ if (upgrade) {
+ store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER);
+ updatedResources.add(modelDesc.getResourcePath());
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ errorMsgs.add("Update DataModelDesc[" + modelDesc.getName() + "] failed: " + e.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
deleted file mode 100644
index 36784de..0000000
--- a/core-job/src/main/java/org/apache/kylin/job/upgrade/CubeDescSignatureUpdate.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-
-package org.apache.kylin.job.upgrade;
-
-import java.util.List;
-
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.project.ProjectManager;
-
-import com.google.common.collect.Lists;
-
-/**
- * Created by dongli on 11/17/15.
- */
-public class CubeDescSignatureUpdate {
- private static final Log logger = LogFactory.getLog(CubeDescSignatureUpdate.class);
- private KylinConfig config = null;
- private ResourceStore store;
- private String[] cubeNames;
- private List<String> updatedResources = Lists.newArrayList();
- private List<String> errorMsgs = Lists.newArrayList();
-
- public CubeDescSignatureUpdate(String[] cubes) {
- config = KylinConfig.getInstanceFromEnv();
- store = ResourceStore.getStore(config);
- cubeNames = cubes;
- }
-
- public static void main(String args[]) {
- if (args != null && args.length > 1) {
- System.out.println("Usage: java CubeDescSignatureUpdate [Cubes]; e.g, cube1,cube2 ");
- return;
- }
-
- CubeDescSignatureUpdate metadataUpgrade = new CubeDescSignatureUpdate(args);
- metadataUpgrade.update();
-
- logger.info("=================================================================");
- logger.info("Run CubeDescSignatureUpdate completed;");
-
- if (!metadataUpgrade.updatedResources.isEmpty()) {
- logger.info("Following resources are updated successfully:");
- for (String s : metadataUpgrade.updatedResources) {
- logger.info(s);
- }
- } else {
- logger.warn("No resource updated.");
- }
-
- if (!metadataUpgrade.errorMsgs.isEmpty()) {
- logger.info("Here are the error/warning messages, you may need to check:");
- for (String s : metadataUpgrade.errorMsgs) {
- logger.warn(s);
- }
- } else {
- logger.info("No error or warning messages; The update succeeds.");
- }
-
- logger.info("=================================================================");
- }
-
- public void update() {
- logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
- CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
- List<CubeDesc> cubeDescs;
- if (ArrayUtils.isEmpty(cubeNames)) {
- cubeDescs = cubeDescManager.listAllDesc();
- } else {
- String[] names = cubeNames[0].split(",");
- if (ArrayUtils.isEmpty(names))
- return;
- cubeDescs = Lists.newArrayListWithCapacity(names.length);
- for (String name : names) {
- cubeDescs.add(cubeDescManager.getCubeDesc(name));
- }
- }
- for (CubeDesc cubeDesc : cubeDescs) {
- updateCubeDesc(cubeDesc);
- }
-
- verify();
- }
-
- private void verify() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
- CubeDescManager.getInstance(config);
- CubeManager.getInstance(config);
- ProjectManager.getInstance(config);
- }
-
- private void updateCubeDesc(CubeDesc cubeDesc) {
- try {
- String calculatedSign = cubeDesc.calculateSignature();
- if (!cubeDesc.getSignature().equals(calculatedSign)) {
- cubeDesc.setSignature(calculatedSign);
- store.putResource(cubeDesc.getResourcePath(), cubeDesc, CubeDescManager.CUBE_DESC_SERIALIZER);
- updatedResources.add(cubeDesc.getResourcePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- errorMsgs.add("Update CubeDesc[" + cubeDesc.getName() + "] failed: " + e.getLocalizedMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/kylin/blob/35052ed3/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
----------------------------------------------------------------------
diff --git a/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java b/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
deleted file mode 100644
index 81b277d..0000000
--- a/core-job/src/main/java/org/apache/kylin/job/upgrade/v2/DataModelDescUpgradeV2.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.kylin.job.upgrade.v2;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.collections4.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.kylin.common.KylinConfig;
-import org.apache.kylin.common.persistence.ResourceStore;
-import org.apache.kylin.cube.CubeDescManager;
-import org.apache.kylin.cube.CubeManager;
-import org.apache.kylin.cube.model.CubeDesc;
-import org.apache.kylin.metadata.MetadataManager;
-import org.apache.kylin.metadata.model.DataModelDesc;
-import org.apache.kylin.metadata.model.DimensionDesc;
-import org.apache.kylin.metadata.model.MeasureDesc;
-import org.apache.kylin.metadata.model.TblColRef;
-import org.apache.kylin.metadata.project.ProjectManager;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-
-/**
- * Created by dongli on 11/30/15.
- *
- * In v1.x, DataModelDesc doesn't include Dimensions and Measures, this tool is to fill them from CubeDesc.
- */
-public class DataModelDescUpgradeV2 {
- private static final Log logger = LogFactory.getLog(DataModelDescUpgradeV2.class);
- private KylinConfig config = null;
- private ResourceStore store;
- private String[] models;
- private List<String> updatedResources = Lists.newArrayList();
- private List<String> errorMsgs = Lists.newArrayList();
-
- public DataModelDescUpgradeV2(String[] models) {
- this.config = KylinConfig.getInstanceFromEnv();
- this.store = ResourceStore.getStore(config);
- this.models = models;
- }
-
- public static void main(String args[]) throws Exception {
- if (args != null && args.length > 1) {
- System.out.println("Usage: java DataModelDescUpradeV2 [Models]; e.g, model1,model2 ");
- return;
- }
-
- DataModelDescUpgradeV2 metadataUpgrade = new DataModelDescUpgradeV2(args);
- metadataUpgrade.upgrade();
-
- logger.info("=================================================================");
- logger.info("Run DataModelDescUpradeV2 completed;");
-
- if (!metadataUpgrade.updatedResources.isEmpty()) {
- logger.info("Following resources are updated successfully:");
- for (String s : metadataUpgrade.updatedResources) {
- logger.info(s);
- }
- } else {
- logger.warn("No resource updated.");
- }
-
- if (!metadataUpgrade.errorMsgs.isEmpty()) {
- logger.info("Here are the error/warning messages, you may need to check:");
- for (String s : metadataUpgrade.errorMsgs) {
- logger.warn(s);
- }
- } else {
- logger.info("No error or warning messages; The update succeeds.");
- }
-
- logger.info("=================================================================");
- }
-
- public void upgrade() {
- logger.info("Reloading Cube Metadata from store: " + store.getReadableResourcePath(ResourceStore.CUBE_DESC_RESOURCE_ROOT));
- CubeDescManager cubeDescManager = CubeDescManager.getInstance(config);
- List<CubeDesc> cubeDescs = cubeDescManager.listAllDesc();
- for (CubeDesc cubeDesc : cubeDescs) {
- if (ArrayUtils.isEmpty(models) || ArrayUtils.contains(models, cubeDesc.getModelName())) {
- upgradeDataModelDesc(cubeDesc);
- }
- }
-
- verify();
- }
-
- private void verify() {
- MetadataManager.getInstance(config).reload();
- CubeDescManager.clearCache();
- CubeDescManager.getInstance(config);
- CubeManager.getInstance(config);
- ProjectManager.getInstance(config);
- }
-
- private void upgradeDataModelDesc(CubeDesc cubeDesc) {
- boolean upgrade = false;
- DataModelDesc modelDesc = cubeDesc.getModel();
- try {
- if (modelDesc != null && modelDesc.getDimensions() == null && modelDesc.getMetrics() == null) {
- List<org.apache.kylin.cube.model.DimensionDesc> cubeDimDescList = cubeDesc.getDimensions();
- if (!CollectionUtils.isEmpty(cubeDimDescList)) {
- Map<String, HashSet<String>> modelDimMap = Maps.newHashMap();
- for (org.apache.kylin.cube.model.DimensionDesc cubeDimDesc : cubeDimDescList) {
- if (!modelDimMap.containsKey(cubeDimDesc.getTable())) {
- modelDimMap.put(cubeDimDesc.getTable(), new HashSet<String>());
- }
- modelDimMap.get(cubeDimDesc.getTable()).addAll(Lists.newArrayList(cubeDimDesc.getDerived() != null ? cubeDimDesc.getDerived() : cubeDimDesc.getColumn()));
- }
-
- List<DimensionDesc> modelDimDescList = Lists.newArrayListWithCapacity(modelDimMap.size());
- for (Map.Entry<String, HashSet<String>> modelDimEntry : modelDimMap.entrySet()) {
- DimensionDesc dimDesc = new DimensionDesc();
- dimDesc.setTable(modelDimEntry.getKey());
- String[] columns = new String[modelDimEntry.getValue().size()];
- columns = modelDimEntry.getValue().toArray(columns);
- dimDesc.setColumns(columns);
- modelDimDescList.add(dimDesc);
- }
- DimensionDesc.capicalizeStrings(modelDimDescList);
- modelDesc.setDimensions(modelDimDescList);
- upgrade = true;
- }
-
- List<MeasureDesc> cubeMeasDescList = cubeDesc.getMeasures();
- if (!CollectionUtils.isEmpty(cubeDimDescList)) {
- ArrayList<String> metrics = Lists.newArrayListWithExpectedSize(cubeMeasDescList.size());
- for (MeasureDesc cubeMeasDesc : cubeMeasDescList) {
- for (TblColRef tblColRef : cubeMeasDesc.getFunction().getParameter().getColRefs()) {
- metrics.add(tblColRef.getName());
- }
- }
- String[] metricsArray = new String[metrics.size()];
- modelDesc.setMetrics(metrics.toArray(metricsArray));
- upgrade = true;
- }
- }
-
- if (upgrade) {
- store.putResource(modelDesc.getResourcePath(), modelDesc, MetadataManager.MODELDESC_SERIALIZER);
- updatedResources.add(modelDesc.getResourcePath());
- }
- } catch (Exception e) {
- e.printStackTrace();
- errorMsgs.add("Update DataModelDesc[" + modelDesc.getName() + "] failed: " + e.getLocalizedMessage());
- }
- }
-}