You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/01/03 11:06:48 UTC
kylin git commit: fix FileSystem close
Repository: kylin
Updated Branches:
refs/heads/fix-sonar-1 676689400 -> a7f0f49e8
fix FileSystem close
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/a7f0f49e
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/a7f0f49e
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/a7f0f49e
Branch: refs/heads/fix-sonar-1
Commit: a7f0f49e8fcdf545ddd72f75f427501b6b333aeb
Parents: 6766894
Author: Billy Liu <bi...@apache.org>
Authored: Tue Jan 3 19:06:22 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Jan 3 19:06:22 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/CliCommandExecutor.java | 15 +++++++++------
.../org/apache/kylin/dict/AppendTrieDictionary.java | 11 ++++++++++-
.../org/apache/kylin/engine/mr/DFSFileTable.java | 1 +
.../java/org/apache/kylin/engine/mr/HadoopUtil.java | 1 +
.../apache/kylin/engine/mr/SortedColumnDFSFile.java | 1 +
.../kylin/engine/mr/common/AbstractHadoopJob.java | 2 ++
.../engine/mr/steps/FactDistinctColumnsReducer.java | 2 ++
.../kylin/engine/mr/steps/MergeStatisticsStep.java | 1 +
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 1 +
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 1 +
.../apache/kylin/engine/spark/SparkCountDemo.java | 2 +-
.../org/apache/kylin/source/hive/HiveMRInput.java | 1 +
.../cardinality/HiveColumnCardinalityUpdateJob.java | 1 +
.../org/apache/kylin/source/kafka/KafkaMRInput.java | 1 +
.../apache/kylin/storage/hbase/HBaseConnection.java | 5 ++++-
.../org/apache/kylin/tool/StorageCleanupJob.java | 2 ++
16 files changed, 39 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index f97b609..488af5f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -131,18 +131,21 @@ public class CliCommandExecutor {
BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
String line;
StringBuilder result = new StringBuilder();
- while ((line = reader.readLine()) != null) {
- result.append(line).append('\n');
- if (logAppender != null) {
- logAppender.log(line);
+ try {
+ while ((line = reader.readLine()) != null) {
+ result.append(line).append('\n');
+ if (logAppender != null) {
+ logAppender.log(line);
+ }
}
- }
- try {
int exitCode = proc.waitFor();
return Pair.newPair(exitCode, result.toString());
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException(e);
+ } finally {
+ reader.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 32bfde6..bdff041 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -160,6 +160,15 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
return Arrays.hashCode(key);
}
+// @Override
+// public boolean equals(Object obj) {
+// if (!(obj instanceof DictSliceKey)){
+// return false;
+// }
+// DictSliceKey other = (DictSliceKey)obj;
+// return Bytes.equals(key, other.key);
+// }
+
@Override
public int compareTo(Object o) {
if (!(o instanceof DictSliceKey)) {
@@ -1219,7 +1228,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
dstFs.delete(dstPath, true);
}
FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf);
-
+ dstFs.close();
return newDict;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
index ee932ac..3358ffc 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/DFSFileTable.java
@@ -105,6 +105,7 @@ public class DFSFileTable implements ReadableTable {
lastModified = Math.max(lastModified, file.getModificationTime());
}
+ fs.close();
return Pair.newPair(size, lastModified);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..55a73ad 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -106,6 +106,7 @@ public class HadoopUtil {
if (fs.exists(path)) {
fs.delete(path, true);
}
+ fs.close();
}
public static byte[] toBytes(Writable writable) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
index 62c309a..27370d3 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/SortedColumnDFSFile.java
@@ -76,6 +76,7 @@ public class SortedColumnDFSFile implements ReadableTable {
DFSFileTableReader reader = new DFSFileTableReader(f.getPath().toString(), -1);
readers.add(reader);
}
+ fs.close();
return new SortedColumnDFSFileReader(readers, comparator);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
index e4e5e82..9a2c391 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/common/AbstractHadoopJob.java
@@ -317,6 +317,8 @@ public abstract class AbstractHadoopJob extends Configured implements Tool {
appendTmpFiles(fileList.toString(), jobConf);
appendTmpJars(jarList.toString(), jobConf);
+ localfs.close();
+ hdfs.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
index 5d42797..a5a2d23 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/FactDistinctColumnsReducer.java
@@ -238,6 +238,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
out.write('\n');
}
} finally {
+ fs.close();
IOUtils.closeQuietly(out);
}
}
@@ -358,6 +359,7 @@ public class FactDistinctColumnsReducer extends KylinReducer<SelfDefineSortableK
}
} finally {
+ fs.close();
IOUtils.closeQuietly(out);
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 811fc24..8227b68 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
e.printStackTrace();
throw e;
} finally {
+ fs.close();
IOUtils.closeStream(reader);
if (tempFile != null)
tempFile.delete();
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..1a4e502 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
String statisticsFileName = newSegment.getStatisticsResourcePath();
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
+ fs.close();
IOUtils.closeStream(is);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dcc9190..dbeec53 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
minValue = Math.min(min, minValue);
maxValue = Math.max(max, maxValue);
} finally {
+ fs.close();
IOUtils.closeQuietly(is);
}
logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
index 6478c10..8a1a42e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
@@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication {
return new Tuple2(key, value);
}
}).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class);
-
+ sc.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..7c7a5d1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setIntermediateTableIdentity(String tableIdentity) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 246822c..b52f204 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
results.add(str);
}
}
+ fileSystem.close();
return results;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..62038db 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setDataPath(String externalDataPath) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..972d637 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -124,6 +124,7 @@ public class HBaseConnection {
coprocessorPool.shutdownNow();
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
coprocessorPool.shutdownNow();
}
}
@@ -205,7 +206,9 @@ public class HBaseConnection {
public static String makeQualifiedPathInHBaseCluster(String path) {
try {
FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
- return fs.makeQualified(new Path(path)).toString();
+ String result = fs.makeQualified(new Path(path)).toString();
+ fs.close();
+ return result;
} catch (IOException e) {
throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/a7f0f49e/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..5b993fb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication {
}
System.out.println("-------------------------------------------------------");
}
+ fs.close();
}
private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception {
@@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication {
} else {
logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path);
}
+ fs.close();
} else {
logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString());
}