You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2020/05/18 11:52:34 UTC
[kylin] 04/07: KYLIN-4414 add a property jobTempDir for
AbstractHadoopJob to indicate the directory for all of the temporary files
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
commit 981cc273be31bb3099c3de34c78f88f986b0db8d
Author: Zhong, Yanghong <nj...@apache.org>
AuthorDate: Tue Mar 10 16:11:34 2020 +0800
KYLIN-4414 add a property jobTempDir for AbstractHadoopJob to indicate the directory for all of the temporary files
---
.../kylin/engine/mr/common/AbstractHadoopJob.java | 48 ++++++++++------------
1 file changed, 21 insertions(+), 27 deletions(-)
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index fd4d413..411caa7 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -157,6 +157,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected OptionsHelper optionsHelper = new OptionsHelper();
protected Job job;
+ private File jobTempDir;
public AbstractHadoopJob() {
super(HadoopUtil.getCurrentConfiguration());
@@ -595,10 +596,7 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected void dumpKylinPropsAndMetadata(String prj, Set<String> dumpList, KylinConfig kylinConfig,
Configuration conf) throws IOException {
- File tmp = File.createTempFile("kylin_job_meta", "");
- FileUtils.forceDelete(tmp); // we need a directory, so delete the file first
-
- File metaDir = new File(tmp, "meta");
+ File metaDir = new File(getJobTempDir(), "meta");
metaDir.mkdirs();
// write kylin.properties
@@ -628,33 +626,29 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
}
protected void cleanupTempConfFile(Configuration conf) {
- String[] tempfiles = StringUtils.split(conf.get("tmpfiles"), ",");
- if (tempfiles == null) {
- return;
- }
- for (String tempMetaFileString : tempfiles) {
- logger.trace("tempMetaFileString is : " + tempMetaFileString);
- if (tempMetaFileString != null) {
- if (tempMetaFileString.startsWith("file://")) {
- tempMetaFileString = tempMetaFileString.substring("file://".length());
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- try {
- FileUtils.forceDelete(tempMetaFile.getParentFile());
-
- } catch (IOException e) {
- logger.warn("error when deleting " + tempMetaFile, e);
- }
- } else {
- logger.info("" + tempMetaFileString + " does not exist");
- }
- } else {
- logger.info("tempMetaFileString is not starting with file:// :" + tempMetaFileString);
- }
+ String tmpFilesString = conf.get("tmpfiles");
+ logger.info("tmpFilesString is : " + tmpFilesString);
+ if (jobTempDir != null) {
+ try {
+ FileUtils.forceDelete(jobTempDir);
+ } catch (IOException e) {
+ logger.warn("error when deleting " + jobTempDir, e);
}
}
}
+ // It's not thread safe
+ protected File getJobTempDir() throws IOException {
+ if (jobTempDir != null && jobTempDir.isDirectory()) {
+ return jobTempDir;
+ }
+ jobTempDir = File.createTempFile("kylin_job_meta", "");
+ FileUtils.forceDelete(jobTempDir); // we need a directory, so delete the file first
+
+ jobTempDir.mkdirs();
+ return jobTempDir;
+ }
+
protected void deletePath(Configuration conf, Path path) throws IOException {
HadoopUtil.deletePath(conf, path);
}