You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2016/12/30 10:28:59 UTC
kylin git commit: fix-2 close FileSystem
Repository: kylin
Updated Branches:
refs/heads/fix-sonar-1 b32435c26 -> c8d95251b
fix-2 close FileSystem
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/c8d95251
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/c8d95251
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/c8d95251
Branch: refs/heads/fix-sonar-1
Commit: c8d95251b1366b59d42b26f6514a99509277c60f
Parents: b32435c
Author: Billy Liu <bi...@apache.org>
Authored: Fri Dec 30 18:28:27 2016 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Fri Dec 30 18:28:27 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java | 1 +
engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java | 1 +
.../main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java | 1 +
.../java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java | 2 ++
.../apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java | 2 ++
.../java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java | 1 +
.../java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java | 1 +
.../apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java | 1 +
.../main/java/org/apache/kylin/engine/spark/SparkCountDemo.java | 2 +-
.../src/main/java/org/apache/kylin/source/hive/HiveMRInput.java | 1 +
.../source/hive/cardinality/HiveColumnCardinalityUpdateJob.java | 1 +
.../src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java | 1 +
tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java | 2 ++
13 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index ee932ac..3358ffc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -105,6 +105,7 @@ public class DFSFileTable implements ReadableTable {
lastModified = Math.max(lastModified, file.getModificationTime());
}
+ fs.close();
return Pair.newPair(size, lastModified);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..55a73ad 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -106,6 +106,7 @@ public class HadoopUtil {
if (fs.exists(path)) {
fs.delete(path, true);
}
+ fs.close();
}
public static byte[] toBytes(Writable writable) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index 62c309a..27370d3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -76,6 +76,7 @@ public class SortedColumnDFSFile implements ReadableTable {
DFSFileTableReader reader = new DFSFileTableReader(f.getPath().toString(), -1);
readers.add(reader);
}
+ fs.close();
return new SortedColumnDFSFileReader(readers, comparator);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index e4e5e82..9a2c391 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -317,6 +317,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
appendTmpFiles(fileList.toString(), jobConf);
appendTmpJars(jarList.toString(), jobConf);
+ localfs.close();
+ hdfs.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5d42797..a5a2d23 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -238,6 +238,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
out.write('\n');
}
} finally {
+ fs.close();
IOUtils.closeQuietly(out);
}
}
@@ -358,6 +359,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
} finally {
+ fs.close();
IOUtils.closeQuietly(out);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 811fc24..8227b68 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
e.printStackTrace();
throw e;
} finally {
+ fs.close();
IOUtils.closeStream(reader);
if (tempFile != null)
tempFile.delete();
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..1a4e502 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
String statisticsFileName = newSegment.getStatisticsResourcePath();
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
+ fs.close();
IOUtils.closeStream(is);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dcc9190..dbeec53 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
minValue = Math.min(min, minValue);
maxValue = Math.max(max, maxValue);
} finally {
+ fs.close();
IOUtils.closeQuietly(is);
}
logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
index 6478c10..8a1a42e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
@@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication {
return new Tuple2(key, value);
}
}).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class);
-
+ sc.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..7c7a5d1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setIntermediateTableIdentity(String tableIdentity) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 246822c..b52f204 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
results.add(str);
}
}
+ fileSystem.close();
return results;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..62038db 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setDataPath(String externalDataPath) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/c8d95251/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..5b993fb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication {
}
System.out.println("-------------------------------------------------------");
}
+ fs.close();
}
private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception {
@@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication {
} else {
logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path);
}
+ fs.close();
} else {
logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString());
}