You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2016/03/23 07:58:51 UTC
[02/50] [abbrv] kylin git commit: KYLIN-1495 bug fixes
KYLIN-1495 bug fixes
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/d52a8caa
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/d52a8caa
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/d52a8caa
Branch: refs/heads/master
Commit: d52a8caa42fd5993742e466175ff6cc201065aee
Parents: 98f8e0a
Author: Hongbin Ma <ma...@apache.org>
Authored: Thu Mar 17 16:20:05 2016 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Thu Mar 17 16:20:05 2016 +0800
----------------------------------------------------------------------
build/bin/upgrade_metadata_v_1_5_1.sh | 41 --------------------
.../kylin/rest/controller/CubeController.java | 23 +++++++----
.../apache/kylin/rest/service/CubeService.java | 6 ++-
.../hbase/cube/v1/CubeSegmentTupleIterator.java | 2 +-
.../hbase/util/DeployCoprocessorCLI.java | 17 +++++---
.../hbase/steps/SandboxMetastoreCLI.java | 2 +-
6 files changed, 33 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/build/bin/upgrade_metadata_v_1_5_1.sh
----------------------------------------------------------------------
diff --git a/build/bin/upgrade_metadata_v_1_5_1.sh b/build/bin/upgrade_metadata_v_1_5_1.sh
deleted file mode 100644
index 6b9701e..0000000
--- a/build/bin/upgrade_metadata_v_1_5_1.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# the script will upgrade Kylin 1.0 ~ 1.3 compatible metadata store to 1.5 compatible metadata store
-# the approach is 1) upgrade to 1.4 compatible 2) upgrade from 1.4 compatible to 1.5 compatible
-
-dir=$(dirname ${0})
-source ${dir}/check-env.sh
-
-# start command
-if [ "$#" -ne 1 ]
-then
- echo "usage: upgrade_metadata_v_1_5_1.sh current_metadata_store_dump_path"
- exit 1
-fi
-
-
-echo "=====Upgrade Cube metadata to 1.4 compatible ====="
-$KYLIN_HOME/bin/kylin.sh org.apache.kylin.cube.upgrade.entry.CubeMetadataUpgradeEntry_v_1_5_1 $1
-
-
-echo "======Deploy coprocessor======="
-$KYLIN_HOME/bin/kylin.sh org.apache.kylin.storage.hbase.util.DeployCoprocessorCLI $KYLIN_HOME/lib/kylin-coprocessor-2.0-SNAPSHOT.jar all
-
-echo "==============End=============="
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
index 410b2bd..6f8ad51 100644
--- a/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
+++ b/server/src/main/java/org/apache/kylin/rest/controller/CubeController.java
@@ -380,19 +380,26 @@ public class CubeController extends BasicController {
// Check if the cube is editable
isCubeDescFreeEditable = cubeService.isCubeDescFreeEditable(desc);
- //cube renaming is not allowed
- if (!cubeRequest.getCubeName().equalsIgnoreCase(CubeService.getCubeNameFromDesc(desc.getName()))) {
- String error = "Cube Desc renaming is not allowed: " + desc.getName();
- updateRequest(cubeRequest, false, error);
- return cubeRequest;
- }
-
String projectName = (null == cubeRequest.getProject()) ? ProjectInstance.DEFAULT_PROJECT_NAME : cubeRequest.getProject();
try {
CubeInstance cube = cubeService.getCubeManager().getCube(cubeRequest.getCubeName());
+
+ if (cube == null) {
+ String error = "The cube named " + cubeRequest.getCubeName() + " does not exist ";
+ updateRequest(cubeRequest, false, error);
+ return cubeRequest;
+ }
+
+ //cube renaming is not allowed
+ if (!cube.getDescriptor().getName().equalsIgnoreCase(desc.getName())) {
+ String error = "Cube Desc renaming is not allowed: desc.getName(): " + desc.getName() + ", cubeRequest.getCubeName(): " + cubeRequest.getCubeName();
+ updateRequest(cubeRequest, false, error);
+ return cubeRequest;
+ }
+
oldCubeDesc = cube.getDescriptor();
if (isCubeDescFreeEditable || oldCubeDesc.consistentWith(desc)) {
- desc = cubeService.updateCubeAndDesc(cube, desc, projectName);
+ desc = cubeService.updateCubeAndDesc(cube, desc, projectName, true);
} else {
logger.warn("Won't update the cube desc due to inconsistency");
updateRequest(cubeRequest, false, "CubeDesc " + desc.getName() + " is inconsistent with existing. Try purge that cube first or avoid updating key cube desc fields.");
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
index 6ef796c..eb15beb 100644
--- a/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
+++ b/server/src/main/java/org/apache/kylin/rest/service/CubeService.java
@@ -244,7 +244,7 @@ public class CubeService extends BasicService {
}
@PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#cube, 'ADMINISTRATION') or hasPermission(#cube, 'MANAGEMENT')")
- public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName) throws IOException, JobException {
+ public CubeDesc updateCubeAndDesc(CubeInstance cube, CubeDesc desc, String newProjectName, boolean forceUpdate) throws IOException, JobException {
final List<CubingJob> cubingJobs = listAllCubingJobs(cube.getName(), null, EnumSet.of(ExecutableState.READY, ExecutableState.RUNNING));
if (!cubingJobs.isEmpty()) {
@@ -252,7 +252,8 @@ public class CubeService extends BasicService {
}
try {
- if (!cube.getDescriptor().consistentWith(desc)) {
+ //double check again
+ if (!forceUpdate && !cube.getDescriptor().consistentWith(desc) ) {
throw new IllegalStateException("cube's desc is not consistent with the new desc");
}
@@ -594,6 +595,7 @@ public class CubeService extends BasicService {
}
private void keepCubeRetention(String cubeName) {
+ logger.info("checking keepCubeRetention");
CubeInstance cube = getCubeManager().getCube(cubeName);
CubeDesc desc = cube.getDescriptor();
if (desc.getRetentionRange() > 0) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
index 909de39..d7ea2a0 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/cube/v1/CubeSegmentTupleIterator.java
@@ -210,7 +210,7 @@ public class CubeSegmentTupleIterator implements ITupleIterator {
private void logScan(HBaseKeyRange keyRange) {
StringBuilder info = new StringBuilder();
- info.append(" Scan hbase table ").append(tableName).append(": ");
+ info.append("Scan hbase table ").append(tableName).append(": ");
if (keyRange.getCuboid().requirePostAggregation()) {
info.append(" cuboid require post aggregation, from ");
} else {
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
index 5bca721..8bf06b7 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/util/DeployCoprocessorCLI.java
@@ -70,21 +70,28 @@ public class DeployCoprocessorCLI {
private static final Logger logger = LoggerFactory.getLogger(DeployCoprocessorCLI.class);
public static void main(String[] args) throws IOException {
+
+ if (args == null || args.length <= 1) {
+ printUsageAndExit();
+ }
+
KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv();
Configuration hconf = HBaseConnection.getCurrentHBaseConfiguration();
FileSystem fileSystem = FileSystem.get(hconf);
HBaseAdmin hbaseAdmin = new HBaseAdmin(hconf);
- String localCoprocessorJar = new File(args[0]).getAbsolutePath();
+ String localCoprocessorJar;
+ if ("default".equals(args[0])) {
+ localCoprocessorJar = kylinConfig.getCoprocessorLocalJar();
+ } else {
+ localCoprocessorJar = new File(args[0]).getAbsolutePath();
+ }
+
logger.info("Identify coprocessor jar " + localCoprocessorJar);
List<String> tableNames = getHTableNames(kylinConfig);
logger.info("Identify tables " + tableNames);
- if (args.length <= 1) {
- printUsageAndExit();
- }
-
String filterType = args[1].toLowerCase();
if (filterType.equals("-table")) {
tableNames = filterByTables(tableNames, Arrays.asList(args).subList(2, args.length));
http://git-wip-us.apache.org/repos/asf/kylin/blob/d52a8caa/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
index 30889ab..b12451a 100644
--- a/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
+++ b/storage-hbase/src/test/java/org/apache/kylin/storage/hbase/steps/SandboxMetastoreCLI.java
@@ -47,7 +47,7 @@ public class SandboxMetastoreCLI {
throw new RuntimeException("No hdp.version set; Please set hdp.version in your jvm option, for example: -Dhdp.version=2.2.4.2-2");
}
- if (args.length < 2) {
+ if (args.length < 1) {
printUsage();
return;
}