You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by sh...@apache.org on 2015/06/19 11:40:03 UTC
incubator-kylin git commit: KYLIN-540 Keep last 30 day’s job history and update the document for metadata migration
Repository: incubator-kylin
Updated Branches:
refs/heads/0.7-staging ad2eee652 -> a487c6e45
KYLIN-540 Keep last 30 day’s job history and update the document for metadata migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/a487c6e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/a487c6e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/a487c6e4
Branch: refs/heads/0.7-staging
Commit: a487c6e452fb841f4421e2a21568c8dce86de463
Parents: ad2eee6
Author: shaofengshi <sh...@apache.org>
Authored: Fri Jun 19 17:39:26 2015 +0800
Committer: shaofengshi <sh...@apache.org>
Committed: Fri Jun 19 17:39:26 2015 +0800
----------------------------------------------------------------------
.../Migrate v0.6.x metadata to v0.7.md | 8 +++++--
.../apache/kylin/job/CubeMetadataUpgrade.java | 22 +++++++++++---------
2 files changed, 18 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a487c6e4/docs/Operations/Migrate v0.6.x metadata to v0.7.md
----------------------------------------------------------------------
diff --git a/docs/Operations/Migrate v0.6.x metadata to v0.7.md b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
index f956107..26140aa 100644
--- a/docs/Operations/Migrate v0.6.x metadata to v0.7.md
+++ b/docs/Operations/Migrate v0.6.x metadata to v0.7.md
@@ -1,4 +1,5 @@
-In v0.7, Kylin refactored the metadata structure, for the new features like inverted-index and streaming; If you have cube created with v0.6 and want to keep in v0.7, a migration is needed; Below is the steps;
+In v0.7, Kylin refactored the metadata structure, for the new features like inverted-index and streaming; If you have cube created with v0.6 and want to keep in v0.7, a migration is needed; (Please skip v0.7.1 as
+it has several compatible issues and the fix will be included in v0.7.2) Below is the steps;
# Backup v0.6 metadata
@@ -24,8 +25,11 @@ This step is to run the migration tool to parse the v0.6 metadata and then conve
hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.job.CubeMetadataUpgrade ./meta_dump
1. The tool will not overwrite v0.6 metadata; It will create a new folder with "_v2" suffix in the same folder, in this case the "./meta_dump_v2" will be created;
-2. By default this tool will not migrate the job history ("/job" and "/job_output");
+2. By default this tool will only migrate the job history in last 30 days; If you want to keep elder job history, please tweak upgradeJobInstance() method by your own;
3. If you see _No error or warning messages; The migration is success_ , that's good; Otherwise please check the error/warning messages carefully;
+4. For some problem you may need manually update the JSON file, to check whether the problem is gone, you can run a verify against the new metadata:
+
+ hbase org.apache.hadoop.util.RunJar ${KYLIN_HOME}/lib/kylin-job-x.x.x-SNAPSHOT-job.jar org.apache.kylin.job.CubeMetadataUpgrade ./meta_dump2 verify
# Upload the new metadata to HBase
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/a487c6e4/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
index 3b40656..b970775 100644
--- a/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
+++ b/job/src/main/java/org/apache/kylin/job/CubeMetadataUpgrade.java
@@ -63,7 +63,6 @@ import org.apache.kylin.metadata.realization.RealizationStatusEnum;
import org.apache.kylin.metadata.realization.RealizationType;
import java.io.File;
-import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
@@ -468,10 +467,18 @@ public class CubeMetadataUpgrade {
for (String path : paths) {
upgradeJobInstance(path);
}
+
+ for (String folder : new String[]{ResourceStore.JOB_PATH_ROOT, ResourceStore.JOB_OUTPUT_PATH_ROOT}) {
+ for (String res : getStore().listResources(folder)) {
+ getStore().deleteResource(res);
+ }
+ getStore().deleteResource(folder);
+ }
} catch (IOException ex) {
errorMsgs.add("upgrade job failed" + ex.getLocalizedMessage());
throw new RuntimeException(ex);
}
+
}
private ExecutableState parseState(JobStatusEnum state) {
@@ -537,7 +544,8 @@ public class CubeMetadataUpgrade {
for (int i = 0, size = job.getSteps().size(); i < size; ++i) {
final JobInstance.JobStep jobStep = job.getSteps().get(i);
- final InputStream inputStream = getStore().getResource(ResourceStore.JOB_OUTPUT_PATH_ROOT + "/" + job.getId() + "." + i);
+ final String outputPath = ResourceStore.JOB_OUTPUT_PATH_ROOT + "/" + job.getId() + "." + i;
+ final InputStream inputStream = getStore().getResource(outputPath);
String output = null;
if (inputStream != null) {
@@ -546,6 +554,7 @@ public class CubeMetadataUpgrade {
if (job_output != null) {
output = job_output.get("output");
}
+ org.apache.commons.io.IOUtils.closeQuietly(inputStream);
}
updateJobStepOutput(jobStep, output, cubingJob.getTasks().get(i));
}
@@ -701,14 +710,7 @@ public class CubeMetadataUpgrade {
String newMetadataUrl = oldMetaFolder.getAbsolutePath() + "_v2";
try {
FileUtils.deleteDirectory(new File(newMetadataUrl));
- FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl), new FileFilter() {
- @Override
- public boolean accept(File pathname) {
- if (pathname.getAbsolutePath().contains(ResourceStore.JOB_PATH_ROOT) || pathname.getAbsolutePath().contains(ResourceStore.JOB_OUTPUT_PATH_ROOT))
- return false;
- return true;
- }
- });
+ FileUtils.copyDirectory(oldMetaFolder, new File(newMetadataUrl));
} catch (IOException e) {
e.printStackTrace();
}