You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/11/08 09:27:26 UTC
[2/3] incubator-kylin git commit: fix migration cli
fix migration cli
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/b66c2580
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/b66c2580
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/b66c2580
Branch: refs/heads/devstreaming
Commit: b66c25803a2f976cca067148278dbe7d7b0d79ef
Parents: 54cab0a
Author: honma <ho...@ebay.com>
Authored: Wed Sep 16 14:37:42 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Wed Sep 16 16:44:03 2015 +0800
----------------------------------------------------------------------
.../org/apache/kylin/common/KylinConfig.java | 2 +-
.../kylin/job/tools/CubeMigrationCLI.java | 19 +++-
.../storage/hbase/steps/HBaseMROutput2.java | 1 +
webapp/app/js/model/cubeDescModel.js | 104 ++++++++++---------
4 files changed, 69 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
index db213f7..43b8c4d 100644
--- a/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
+++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfig.java
@@ -502,7 +502,7 @@ public class KylinConfig implements Serializable {
}
public String getHbaseDefaultCompressionCodec() {
- return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC);
+ return getOptional(HTABLE_DEFAULT_COMPRESSION_CODEC,"");
}
public boolean isHiveKeepFlatTable() {
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
index 89d55f6..c68196a 100644
--- a/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
+++ b/job/src/main/java/org/apache/kylin/job/tools/CubeMigrationCLI.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -45,6 +46,7 @@ import org.apache.kylin.dict.lookup.SnapshotManager;
import org.apache.kylin.dict.lookup.SnapshotTable;
import org.apache.kylin.job.JobInstance;
import org.apache.kylin.metadata.model.DataModelDesc;
+import org.apache.kylin.metadata.model.IEngineAware;
import org.apache.kylin.metadata.model.SegmentStatusEnum;
import org.apache.kylin.metadata.model.TableDesc;
import org.apache.kylin.metadata.project.ProjectInstance;
@@ -87,7 +89,6 @@ public class CubeMigrationCLI {
private static void usage() {
System.out.println("Usage: CubeMigrationCLI srcKylinConfigUri dstKylinConfigUri cubeName projectName overwriteIfExists realExecute");
System.out.println(" srcKylinConfigUri: The KylinConfig of the cube’s source \n" + "dstKylinConfigUri: The KylinConfig of the cube’s new home \n" + "cubeName: the name of cube to be migrated. \n" + "projectName: The target project in the target environment.(Make sure it exist) \n" + "overwriteIfExists: overwrite cube if it already exists in the target environment. \n" + "realExecute: if false, just print the operations to take, if true, do the real migration. \n");
-
}
public static void moveCube(KylinConfig srcCfg, KylinConfig dstCfg, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
@@ -132,7 +133,6 @@ public class CubeMigrationCLI {
}
public static void moveCube(String srcCfgUri, String dstCfgUri, String cubeName, String projectName, String overwriteIfExists, String realExecute) throws IOException, InterruptedException {
-
moveCube(KylinConfig.createInstanceFromUri(srcCfgUri), KylinConfig.createInstanceFromUri(dstCfgUri), cubeName, projectName, overwriteIfExists, realExecute);
}
@@ -143,8 +143,8 @@ public class CubeMigrationCLI {
logger.info("src metadata url is " + srcMetadataUrl);
logger.info("dst metadata url is " + dstMetadataUrl);
- int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase:");
- int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase:");
+ int srcIndex = srcMetadataUrl.toLowerCase().indexOf("hbase");
+ int dstIndex = dstMetadataUrl.toLowerCase().indexOf("hbase");
if (srcIndex < 0 || dstIndex < 0)
throw new IllegalStateException("Both metadata urls should be hbase metadata url");
@@ -162,6 +162,11 @@ public class CubeMigrationCLI {
for (CubeSegment segment : cube.getSegments()) {
String jobUuid = segment.getLastBuildJobID();
+
+ if (StringUtils.isEmpty(jobUuid)) {
+ //segments build from streaming does not have hdfs working dir
+ continue;
+ }
String src = JobInstance.getJobWorkingDir(jobUuid, srcConfig.getHdfsWorkingDirectory());
String tgt = JobInstance.getJobWorkingDir(jobUuid, dstConfig.getHdfsWorkingDirectory());
@@ -216,7 +221,11 @@ public class CubeMigrationCLI {
for (CubeSegment segment : cube.getSegments()) {
dictAndSnapshot.addAll(segment.getSnapshotPaths());
dictAndSnapshot.addAll(segment.getDictionaryPaths());
- metaResource.add(segment.getStatisticsResourcePath());
+
+ if (cube.getDescriptor().getEngineType() == IEngineAware.ID_MR_V2) {
+ //only V2 has this
+ metaResource.add(segment.getStatisticsResourcePath());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
index 1e414be..79ef403 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/steps/HBaseMROutput2.java
@@ -170,6 +170,7 @@ public class HBaseMROutput2 implements IMROutput2 {
scans.add(scan);
}
TableMapReduceUtil.initTableMapperJob(scans, (Class<? extends TableMapper>) mapper, outputKeyClass, outputValueClass, job);
+ TableMapReduceUtil.initCredentials(job);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/b66c2580/webapp/app/js/model/cubeDescModel.js
----------------------------------------------------------------------
diff --git a/webapp/app/js/model/cubeDescModel.js b/webapp/app/js/model/cubeDescModel.js
index 86c8444..3376bfc 100644
--- a/webapp/app/js/model/cubeDescModel.js
+++ b/webapp/app/js/model/cubeDescModel.js
@@ -14,62 +14,64 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
-*/
+ */
-KylinApp.service('CubeDescModel',function(){
+KylinApp.service('CubeDescModel', function () {
- this.cubeMetaFrame = {};
+ this.cubeMetaFrame = {};
- //
- this.createNew = function () {
- var cubeMeta = {
- "name": "",
- "model_name": "",
- "description": "",
- "dimensions": [],
- "measures": [
- { "id": 1,
- "name": "_COUNT_",
- "function": {
- "expression": "COUNT",
- "returntype": "bigint",
- "parameter": {
- "type": "constant",
- "value": "1"
- }
- }
- }
- ],
- "rowkey": {
- "rowkey_columns": [],
- "aggregation_groups": []
- },
- "notify_list": [],
- "hbase_mapping": {
- "column_family": []
- },
- "retention_range":"0",
- "auto_merge_time_ranges":[604800000,2419200000]
- };
-
- return cubeMeta;
- };
+ //
+ this.createNew = function () {
+ var cubeMeta = {
+ "name": "",
+ "model_name": "",
+ "description": "",
+ "dimensions": [],
+ "measures": [
+ {
+ "id": 1,
+ "name": "_COUNT_",
+ "function": {
+ "expression": "COUNT",
+ "returntype": "bigint",
+ "parameter": {
+ "type": "constant",
+ "value": "1"
+ }
+ }
+ }
+ ],
+ "rowkey": {
+ "rowkey_columns": [],
+ "aggregation_groups": []
+ },
+ "notify_list": [],
+ "hbase_mapping": {
+ "column_family": []
+ },
+ "retention_range": "0",
+ "auto_merge_time_ranges": [604800000, 2419200000],
+ "engine_type": 2
+ };
- this.createMeasure = function (){
- var measure = {
- "id": "",
- "name": "",
- "function": {
- "expression": "",
- "returntype": "",
- "parameter": {
- "type": "",
- "value": ""
- }
- }
- };
+ return cubeMeta;
+ };
- return measure;
+ this.createMeasure = function () {
+ var measure = {
+ "id": "",
+ "name": "",
+ "function": {
+ "expression": "",
+ "returntype": "",
+ "parameter": {
+ "type": "",
+ "value": ""
}
+ }
+ };
+
+ return measure;
+ }
})