You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by bi...@apache.org on 2017/01/03 12:37:25 UTC
kylin git commit: fix FileSystem.close [Forced Update!]
Repository: kylin
Updated Branches:
refs/heads/fix-sonar-1 a7f0f49e8 -> 7972ac5bb (forced update)
fix FileSystem.close
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo
Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/7972ac5b
Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/7972ac5b
Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/7972ac5b
Branch: refs/heads/fix-sonar-1
Commit: 7972ac5bb86aee57127eeafa6cee7ca3b5548cd9
Parents: 6766894
Author: Billy Liu <bi...@apache.org>
Authored: Tue Jan 3 20:37:13 2017 +0800
Committer: Billy Liu <bi...@apache.org>
Committed: Tue Jan 3 20:37:13 2017 +0800
----------------------------------------------------------------------
.../apache/kylin/common/util/CliCommandExecutor.java | 15 +++++++++------
.../org/apache/kylin/dict/AppendTrieDictionary.java | 11 ++++++++++-
.../java/org/apache/kylin/engine/mr/HadoopUtil.java | 1 +
.../kylin/engine/mr/steps/MergeStatisticsStep.java | 1 +
.../kylin/engine/mr/steps/SaveStatisticsStep.java | 1 +
.../mr/steps/UpdateCubeInfoAfterBuildStep.java | 1 +
.../apache/kylin/engine/spark/SparkCountDemo.java | 2 +-
.../org/apache/kylin/source/hive/HiveMRInput.java | 1 +
.../cardinality/HiveColumnCardinalityUpdateJob.java | 1 +
.../org/apache/kylin/source/kafka/KafkaMRInput.java | 1 +
.../apache/kylin/storage/hbase/HBaseConnection.java | 5 ++++-
.../org/apache/kylin/tool/StorageCleanupJob.java | 2 ++
12 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
----------------------------------------------------------------------
diff --git a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
index f97b609..488af5f 100644
--- a/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
+++ b/core-common/src/main/java/org/apache/kylin/common/util/CliCommandExecutor.java
@@ -131,18 +131,21 @@ public class CliCommandExecutor {
BufferedReader reader = new BufferedReader(new InputStreamReader(proc.getInputStream()));
String line;
StringBuilder result = new StringBuilder();
- while ((line = reader.readLine()) != null) {
- result.append(line).append('\n');
- if (logAppender != null) {
- logAppender.log(line);
+ try {
+ while ((line = reader.readLine()) != null) {
+ result.append(line).append('\n');
+ if (logAppender != null) {
+ logAppender.log(line);
+ }
}
- }
- try {
int exitCode = proc.waitFor();
return Pair.newPair(exitCode, result.toString());
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
throw new IOException(e);
+ } finally {
+ reader.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
----------------------------------------------------------------------
diff --git a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
index 32bfde6..1d17569 100644
--- a/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
+++ b/core-dictionary/src/main/java/org/apache/kylin/dict/AppendTrieDictionary.java
@@ -161,6 +161,15 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
}
@Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof DictSliceKey)){
+ return false;
+ }
+ DictSliceKey other = (DictSliceKey)obj;
+ return Bytes.equals(key, other.key);
+ }
+
+ @Override
public int compareTo(Object o) {
if (!(o instanceof DictSliceKey)) {
return -1;
@@ -1219,7 +1228,7 @@ public class AppendTrieDictionary<T> extends Dictionary<T> {
dstFs.delete(dstPath, true);
}
FileUtil.copy(FileSystem.get(srcPath.toUri(), conf), srcPath, FileSystem.get(dstPath.toUri(), conf), dstPath, false, true, conf);
-
+ dstFs.close();
return newDict;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
index 88692a0..55a73ad 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/HadoopUtil.java
@@ -106,6 +106,7 @@ public class HadoopUtil {
if (fs.exists(path)) {
fs.delete(path, true);
}
+ fs.close();
}
public static byte[] toBytes(Writable writable) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
index 811fc24..8227b68 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/MergeStatisticsStep.java
@@ -115,6 +115,7 @@ public class MergeStatisticsStep extends AbstractExecutable {
e.printStackTrace();
throw e;
} finally {
+ fs.close();
IOUtils.closeStream(reader);
if (tempFile != null)
tempFile.delete();
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
index 020c62c..1a4e502 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/SaveStatisticsStep.java
@@ -70,6 +70,7 @@ public class SaveStatisticsStep extends AbstractExecutable {
String statisticsFileName = newSegment.getStatisticsResourcePath();
rs.putResource(statisticsFileName, is, System.currentTimeMillis());
} finally {
+ fs.close();
IOUtils.closeStream(is);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
----------------------------------------------------------------------
diff --git a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
index dcc9190..dbeec53 100644
--- a/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
+++ b/engine-mr/src/main/java/org/apache/kylin/engine/mr/steps/UpdateCubeInfoAfterBuildStep.java
@@ -91,6 +91,7 @@ public class UpdateCubeInfoAfterBuildStep extends AbstractExecutable {
minValue = Math.min(min, minValue);
maxValue = Math.max(max, maxValue);
} finally {
+ fs.close();
IOUtils.closeQuietly(is);
}
logger.info("updateTimeRange step. minValue:" + minValue + " maxValue:" + maxValue);
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
----------------------------------------------------------------------
diff --git a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
index 6478c10..8a1a42e 100644
--- a/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
+++ b/engine-spark/src/main/java/org/apache/kylin/engine/spark/SparkCountDemo.java
@@ -75,6 +75,6 @@ public class SparkCountDemo extends AbstractApplication {
return new Tuple2(key, value);
}
}).saveAsNewAPIHadoopFile("hdfs://10.249.65.231:8020/tmp/hfile", ImmutableBytesWritable.class, KeyValue.class, HFileOutputFormat.class);
-
+ sc.close();
}
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
index fc2b982..7c7a5d1 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/HiveMRInput.java
@@ -383,6 +383,7 @@ public class HiveMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setIntermediateTableIdentity(String tableIdentity) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
----------------------------------------------------------------------
diff --git a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
index 246822c..b52f204 100644
--- a/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
+++ b/source-hive/src/main/java/org/apache/kylin/source/hive/cardinality/HiveColumnCardinalityUpdateJob.java
@@ -155,6 +155,7 @@ public class HiveColumnCardinalityUpdateJob extends AbstractHadoopJob {
results.add(str);
}
}
+ fileSystem.close();
return results;
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
----------------------------------------------------------------------
diff --git a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
index dde3584..62038db 100644
--- a/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
+++ b/source-kafka/src/main/java/org/apache/kylin/source/kafka/KafkaMRInput.java
@@ -227,6 +227,7 @@ public class KafkaMRInput implements IMRInput {
if (fs.exists(externalDataPath)) {
fs.delete(externalDataPath, true);
}
+ fs.close();
}
public void setDataPath(String externalDataPath) {
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
----------------------------------------------------------------------
diff --git a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
index cbf81b6..972d637 100644
--- a/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
+++ b/storage-hbase/src/main/java/org/apache/kylin/storage/hbase/HBaseConnection.java
@@ -124,6 +124,7 @@ public class HBaseConnection {
coprocessorPool.shutdownNow();
}
} catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
coprocessorPool.shutdownNow();
}
}
@@ -205,7 +206,9 @@ public class HBaseConnection {
public static String makeQualifiedPathInHBaseCluster(String path) {
try {
FileSystem fs = FileSystem.get(getCurrentHBaseConfiguration());
- return fs.makeQualified(new Path(path)).toString();
+ String result = fs.makeQualified(new Path(path)).toString();
+ fs.close();
+ return result;
} catch (IOException e) {
throw new IllegalArgumentException("Cannot create FileSystem from current hbase cluster conf", e);
}
http://git-wip-us.apache.org/repos/asf/kylin/blob/7972ac5b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
----------------------------------------------------------------------
diff --git a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
index 05e0142..5b993fb 100644
--- a/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
+++ b/tool/src/main/java/org/apache/kylin/tool/StorageCleanupJob.java
@@ -243,6 +243,7 @@ public class StorageCleanupJob extends AbstractApplication {
}
System.out.println("-------------------------------------------------------");
}
+ fs.close();
}
private void cleanUnusedIntermediateHiveTable(Configuration conf) throws Exception {
@@ -349,6 +350,7 @@ public class StorageCleanupJob extends AbstractApplication {
} else {
logger.info("Hive table {}'s external path {} not exist. It's normal if kylin.source.hive.keep-flat-table set false (By default)", tableToDelete, path);
}
+ fs.close();
} else {
logger.warn("Hive table {}'s job ID not found, segmentId2JobId: {}", tableToDelete, segmentId2JobId.toString());
}