You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by li...@apache.org on 2017/03/12 11:48:44 UTC
[12/48] kylin git commit: minor, adopt json parse in CubeMetaExtractor
minor, adopt json parse in CubeMetaExtractor
Signed-off-by: Hongbin Ma <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/6c1f53ec
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/6c1f53ec
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/6c1f53ec
Branch: refs/heads/master-hbase0.98
Commit: 6c1f53ec9cdce3730b4c7262538dba698ddb5628
Parents: 7d57163
Author: Roger Shi <ro...@hotmail.com>
Authored: Wed Mar 1 12:08:06 2017 +0800
Committer: Hongbin Ma <ma...@apache.org>
Committed: Wed Mar 1 13:53:36 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/tool/CubeMetaExtractor.java | 38 +++++++++++++-------
1 file changed, 25 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/6c1f53ec/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
index 30f9736..35831f7 100644
--- a/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
+++ b/tool/src/main/java/org/apache/kylin/tool/CubeMetaExtractor.java
@@ -23,10 +23,11 @@ import java.io.IOException;
import java.util.List;
import java.util.Set;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.SerializationFeature;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.OptionGroup;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.persistence.ResourceStore;
@@ -60,6 +61,9 @@ import org.apache.kylin.storage.hybrid.HybridManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
@@ -78,10 +82,10 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
private static final Option OPTION_PROJECT = OptionBuilder.withArgName("project").hasArg().isRequired(false).withDescription("Specify realizations in which project to extract").create("project");
@SuppressWarnings("static-access")
- private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageEngine").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageEngine");
+ private static final Option OPTION_STORAGE_TYPE = OptionBuilder.withArgName("storageType").hasArg().isRequired(false).withDescription("Specify the storage type to overwrite. Default is empty, keep origin.").create("storageType");
@SuppressWarnings("static-access")
- private static final Option OPTION_ENGINE_TYPE = OptionBuilder.withArgName("typeEngine").hasArg().isRequired(false).withDescription("Specify the engine type to overwrite. Default is empty, keep origin.").create("typeEngine");
+ private static final Option OPTION_ENGINE_TYPE = OptionBuilder.withArgName("engineType").hasArg().isRequired(false).withDescription("Specify the engine type to overwrite. Default is empty, keep origin.").create("engineType");
@SuppressWarnings("static-access")
private static final Option OPTION_INCLUDE_SEGMENTS = OptionBuilder.withArgName("includeSegments").hasArg().isRequired(false).withDescription("set this to true if want extract the segments info. Default true").create("includeSegments");
@@ -143,8 +147,8 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
includeJobs = optionsHelper.hasOption(OPTION_INCLUDE_JOB) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_JOB)) : false;
includeSegmentDetails = optionsHelper.hasOption(OPTION_INCLUDE_SEGMENT_DETAILS) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_SEGMENT_DETAILS)) : false;
onlyJobOutput = optionsHelper.hasOption(OPTION_INCLUDE_ONLY_JOB_OUTPUT) ? Boolean.valueOf(optionsHelper.getOptionValue(OPTION_INCLUDE_ONLY_JOB_OUTPUT)) : true;
- storageType = optionsHelper.hasOption(OPTION_STORAGE_TYPE) ? optionsHelper.getOptionValue(OPTION_STORAGE_TYPE): null;
- engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE): null;
+ storageType = optionsHelper.hasOption(OPTION_STORAGE_TYPE) ? optionsHelper.getOptionValue(OPTION_STORAGE_TYPE) : null;
+ engineType = optionsHelper.hasOption(OPTION_ENGINE_TYPE) ? optionsHelper.getOptionValue(OPTION_ENGINE_TYPE) : null;
kylinConfig = KylinConfig.getInstanceFromEnv();
metadataManager = MetadataManager.getInstance(kylinConfig);
@@ -253,17 +257,25 @@ public class CubeMetaExtractor extends AbstractInfoExtractor {
}
private void engineOverwriteInternal(File f) throws IOException {
- List<String> lines = FileUtils.readLines(f, "UTF-8");
- for (int i = 0, n = lines.size(); i < n; i++) {
- String l = lines.get(i);
- if (l.contains("\"engine_type\"")) {
- lines.set(i, " \"engine_type\" : " + engineType + ",");
+ try {
+ ObjectMapper objectMapper = new ObjectMapper();
+ JsonNode rootNode = objectMapper.readTree(f);
+ boolean replaced = false;
+ if (engineType != null && rootNode.get("engine_type") != null) {
+ ((ObjectNode) rootNode).put("engine_type", Integer.parseInt(engineType));
+ replaced = true;
+ }
+ if (storageType != null && rootNode.get("storage_type") != null) {
+ ((ObjectNode) rootNode).put("storage_type", Integer.parseInt(storageType));
+ replaced = true;
}
- if (l.contains("\"storage_type\"")) {
- lines.set(i, " \"storage_type\" : " + storageType + ",");
+ if (replaced) {
+ objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
+ objectMapper.writeValue(f, rootNode);
}
+ } catch (JsonProcessingException ex) {
+ logger.info("cannot parse file {}", f);
}
- FileUtils.writeLines(f, "UTF-8", lines);
}
private IRealization getRealization(RealizationEntry realizationEntry) {