You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ma...@apache.org on 2015/08/17 12:35:04 UTC
incubator-kylin git commit: KYLIN-926 remove intermediate files
Repository: incubator-kylin
Updated Branches:
refs/heads/KYLIN-926-08 [created] ad15c40fc
KYLIN-926 remove intermediate files
Project: http://git-wip-us.apache.org/repos/asf/incubator-kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kylin/commit/ad15c40f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kylin/tree/ad15c40f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kylin/diff/ad15c40f
Branch: refs/heads/KYLIN-926-08
Commit: ad15c40fca8b2c1edbe6227c10a4ad88674843d3
Parents: 3826896
Author: honma <ho...@ebay.com>
Authored: Mon Aug 17 17:18:36 2015 +0800
Committer: honma <ho...@ebay.com>
Committed: Mon Aug 17 18:35:38 2015 +0800
----------------------------------------------------------------------
.../apache/kylin/engine/mr/common/AbstractHadoopJob.java | 11 ++++++++---
.../apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 2 +-
.../engine/mr/steps/FactDistinctColumnsReducerTest.java | 1 +
.../apache/kylin/job/streaming/CubeStreamConsumer.java | 2 +-
.../java/org/apache/kylin/job/BuildIIWithStreamTest.java | 3 ++-
.../apache/kylin/storage/hbase/HBaseStreamingOutput.java | 2 +-
.../hbase/coprocessor/endpoint/CubeVisitService.java | 1 +
7 files changed, 15 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index fffaf36..36e394c 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -288,9 +288,14 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
protected void cleanupTempConfFile(Configuration conf) {
String tempMetaFileString = conf.get("tmpfiles");
if (tempMetaFileString != null) {
- File tempMetaFile = new File(tempMetaFileString);
- if (tempMetaFile.exists()) {
- tempMetaFile.getParentFile().delete();
+ if (tempMetaFileString.startsWith("file:///")) {
+ tempMetaFileString = tempMetaFileString.substring("file:///".length());
+ File tempMetaFile = new File(tempMetaFileString);
+ if (tempMetaFile.exists()) {
+ tempMetaFile.getParentFile().delete();
+ }
+ } else {
+ logger.warn("tempMetaFileString is not starting with file:/// :" + tempMetaFileString);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 3757352..2fe4dcf 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -71,7 +71,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
IOUtils.closeStream(is);
- fs.delete(statisticsFilePath, false);
+ fs.delete(statisticsFilePath, true);
}
return new ExecuteResult(ExecuteResult.State.SUCCEED, "succeed");
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
index edfc5c0..165bc13 100644
--- a/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
+++ b/engine-mr/src/test/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducerTest.java
@@ -29,6 +29,7 @@ public class FactDistinctColumnsReducerTest {
System.out.println(outputPath);
Map<Long, HyperLogLogPlusCounter> cuboidHLLMap = Maps.newHashMap();
FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, cuboidHLLMap, 100);
+ FileSystem.getLocal(conf).delete(outputPath, true);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
----------------------------------------------------------------------
diff --git a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
index 4a909ac..7ba275d 100644
--- a/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
+++ b/job/src/main/java/org/apache/kylin/job/streaming/CubeStreamConsumer.java
@@ -101,11 +101,11 @@ public class CubeStreamConsumer implements MicroStreamBatchConsumer {
final Configuration conf = HadoopUtil.getCurrentConfiguration();
final Path outputPath = new Path("file:///tmp/cuboidstatistics/" + UUID.randomUUID().toString());
- FileSystem.getLocal(conf).deleteOnExit(outputPath);
FactDistinctColumnsReducer.writeCuboidStatistics(conf, outputPath, samplingResult, 100);
FSDataInputStream localStream = FileSystem.getLocal(conf).open(new Path(outputPath, BatchConstants.CFG_STATISTICS_CUBOID_ESTIMATION));
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), localStream, System.currentTimeMillis());
localStream.close();
+ FileSystem.getLocal(conf).delete(outputPath, true);
final Map<TblColRef, Dictionary<?>> dictionaryMap = CubingUtils.buildDictionary(cubeInstance, parsedStreamMessages);
Map<TblColRef, Dictionary<?>> realDictMap = writeDictionary(cubeSegment, dictionaryMap, startOffset, endOffset);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
----------------------------------------------------------------------
diff --git a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
index 8357e1b..0c57e4b 100644
--- a/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
+++ b/job/src/test/java/org/apache/kylin/job/BuildIIWithStreamTest.java
@@ -55,6 +55,7 @@ import org.apache.kylin.common.KylinConfig;
import org.apache.kylin.common.util.AbstractKylinTestCase;
import org.apache.kylin.common.util.ClassUtil;
import org.apache.kylin.common.util.DateFormat;
+import org.apache.kylin.engine.mr.JobBuilderSupport;
import org.apache.kylin.invertedindex.IIInstance;
import org.apache.kylin.invertedindex.IIManager;
import org.apache.kylin.invertedindex.IISegment;
@@ -119,7 +120,7 @@ public class BuildIIWithStreamTest {
JobEngineConfig jobEngineConfig = new JobEngineConfig(kylinConfig);
final String uuid = UUID.randomUUID().toString();
final String dropTableHql = JoinedFlatTable.generateDropTableStatement(intermediateTableDesc);
- final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, jobEngineConfig.getHdfsWorkingDirectory() + "/kylin-" + uuid);
+ final String createTableHql = JoinedFlatTable.generateCreateTableStatement(intermediateTableDesc, JobBuilderSupport.getJobWorkingDir(jobEngineConfig,uuid));
String insertDataHqls;
try {
insertDataHqls = JoinedFlatTable.generateInsertDataStatement(intermediateTableDesc, jobEngineConfig);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
index 608acf4..7a67183 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseStreamingOutput.java
@@ -73,7 +73,7 @@ public class HBaseStreamingOutput implements IStreamingOutput {
ResourceStore.getStore(kylinConfig).putResource(cubeSegment.getStatisticsResourcePath(), inputStream, System.currentTimeMillis());
} finally {
IOUtils.closeQuietly(inputStream);
- FileSystem.getLocal(conf).delete(outputPath, false);
+ FileSystem.getLocal(conf).delete(outputPath, true);
}
} catch (IOException e) {
throw new RuntimeException("failed to write sampling result", e);
http://git-wip-us.apache.org/repos/asf/incubator-kylin/blob/ad15c40f/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
----------------------------------------------------------------------
diff --git a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
index 3ece09e..2ec59fa 100644
--- a/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
+++ b/storage/src/main/java/org/apache/kylin/storage/hbase/coprocessor/endpoint/CubeVisitService.java
@@ -130,6 +130,7 @@ public class CubeVisitService extends CubeVisitProtos.CubeVisitService implement
GTScanRequest scanReq = KryoUtils.deserialize(request.getGtScanRequest().toByteArray(), GTScanRequest.class);
RawScan hbaseRawScan = KryoUtils.deserialize(request.getHbaseRawScan().toByteArray(), RawScan.class);
+ //TODO: rewrite own start/end
Scan scan = CubeHBaseRPC.buildScan(hbaseRawScan);
region = env.getRegion();