You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/06/04 15:25:22 UTC
incubator-ignite git commit: #[IGNITE-218]: fixed by review results.
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-218 0fa49ed28 -> b347c4b32
#[IGNITE-218]: fixed by review results.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/b347c4b3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/b347c4b3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/b347c4b3
Branch: refs/heads/ignite-218
Commit: b347c4b325df9717bb2d87780a13609109f826db
Parents: 0fa49ed
Author: iveselovskiy <iv...@gridgain.com>
Authored: Thu Jun 4 16:20:46 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Thu Jun 4 16:20:46 2015 +0300
----------------------------------------------------------------------
.../hadoop/fs/IgniteHadoopFileSystemCounterWriter.java | 1 -
.../ignite/internal/processors/hadoop/HadoopUtils.java | 11 +++++------
.../internal/processors/hadoop/v2/HadoopV2Job.java | 4 +---
.../processors/hadoop/v2/HadoopV2JobResourceManager.java | 11 ++++-------
4 files changed, 10 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
index 2305f1e..d910507 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopFileSystemCounterWriter.java
@@ -72,7 +72,6 @@ public class IgniteHadoopFileSystemCounterWriter implements HadoopCounterWriter
try {
hadoopCfg.set(MRJobConfig.USER_NAME, user);
- // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
FileSystem fs = HadoopUtils.fileSystemForMrUser(jobStatPath.toUri(), hadoopCfg, true);
fs.mkdirs(jobStatPath);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
index d419915..68a9ef6 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopUtils.java
@@ -64,7 +64,6 @@ public class HadoopUtils {
private static final String OLD_REDUCE_CLASS_ATTR = "mapred.reducer.class";
/** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
- // TODO: cleam up this cache upon Ignite node stop, see https://issues.apache.org/jira/browse/IGNITE-980 .
private static final HadoopLazyConcurrentMap<FsCacheKey, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
new HadoopLazyConcurrentMap.ValueFactory<FsCacheKey, FileSystem>() {
@Override public FileSystem createValue(FsCacheKey key) {
@@ -76,12 +75,14 @@ public class HadoopUtils {
String scheme = uri.getScheme();
+ // Copy the configuration to avoid altering the external object.
+ Configuration cfg = new Configuration(key.configuration());
+
String prop = HadoopUtils.disableFsCachePropertyName(scheme);
- // TODO: Copy configuration instead of altering existing one.
- key.configuration().setBoolean(prop, true);
+ cfg.setBoolean(prop, true);
- return FileSystem.get(uri, key.configuration(), key.user());
+ return FileSystem.get(uri, cfg, key.user());
}
catch (IOException | InterruptedException ioe) {
throw new IgniteException(ioe);
@@ -166,8 +167,6 @@ public class HadoopUtils {
break;
case PHASE_REDUCE:
- // TODO: temporary fixed, but why PHASE_REDUCE could have 0 reducers?
- // See https://issues.apache.org/jira/browse/IGNITE-764
setupProgress = 1;
mapProgress = 1;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
index 849bbe6..d754039 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2Job.java
@@ -134,7 +134,6 @@ public class HadoopV2Job implements HadoopJob {
Path jobDir = new Path(jobDirPath);
try {
- // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
FileSystem fs = fileSystemForMrUser(jobDir.toUri(), jobConf, true);
JobSplit.TaskSplitMetaInfo[] metaInfos = SplitMetaInfoReader.readSplitMetaInfo(hadoopJobID, fs, jobConf,
@@ -322,8 +321,7 @@ public class HadoopV2Job implements HadoopJob {
/** {@inheritDoc} */
@Override public void cleanupStagingDirectory() {
- if (rsrcMgr != null)
- rsrcMgr.cleanupStagingDirectory();
+ rsrcMgr.cleanupStagingDirectory();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/b347c4b3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
index d34d250..2f64e77 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/v2/HadoopV2JobResourceManager.java
@@ -40,6 +40,9 @@ import java.util.*;
* files are needed to be placed on local files system.
*/
public class HadoopV2JobResourceManager {
+ /** File type Fs disable caching property name. */
+ private static final String FILE_DISABLE_CACHING_PROPERTY_NAME = HadoopUtils.disableFsCachePropertyName("file");
+
/** Hadoop job context. */
private final JobContextImpl ctx;
@@ -84,10 +87,7 @@ public class HadoopV2JobResourceManager {
try {
cfg.set(HadoopFileSystemsUtils.LOC_FS_WORK_DIR_PROP, dir.getAbsolutePath());
- // TODO: Make constant.
- String prop = HadoopUtils.disableFsCachePropertyName("file");
-
- if (!cfg.getBoolean(prop, false))
+ if (!cfg.getBoolean(FILE_DISABLE_CACHING_PROPERTY_NAME, false))
FileSystem.getLocal(cfg).setWorkingDirectory(new Path(dir.getAbsolutePath()));
}
finally {
@@ -115,7 +115,6 @@ public class HadoopV2JobResourceManager {
stagingDir = new Path(new URI(mrDir));
if (download) {
- // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
FileSystem fs = HadoopUtils.fileSystemForMrUser(stagingDir.toUri(), cfg, true);
if (!fs.exists(stagingDir))
@@ -211,7 +210,6 @@ public class HadoopV2JobResourceManager {
FileSystem dstFs = FileSystem.getLocal(cfg);
- // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
FileSystem srcFs = HadoopUtils.fileSystemForMrUser(srcPath.toUri(), cfg, true);
if (extract) {
@@ -295,7 +293,6 @@ public class HadoopV2JobResourceManager {
public void cleanupStagingDirectory() {
try {
if (stagingDir != null)
- // TODO: Check if FileSystem should be closed, see https://issues.apache.org/jira/browse/IGNITE-980
HadoopUtils.fileSystemForMrUser(stagingDir.toUri(), ctx.getJobConf(), true).delete(stagingDir, true);
}
catch (Exception e) {