You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2016/08/02 07:41:26 UTC
[03/14] ignite git commit: IGNITE-3343: test fixes,
rollback unnecessary changes.
IGNITE-3343: test fixes, rollback unnecessary changes.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e7dfc178
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e7dfc178
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e7dfc178
Branch: refs/heads/ignite-3343
Commit: e7dfc178d88337e462e678f945d1ac48227d9b72
Parents: 69fb90b
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Jul 26 13:12:03 2016 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Jul 26 13:12:03 2016 +0300
----------------------------------------------------------------------
.../org/apache/ignite/IgniteFileSystem.java | 2 +-
.../internal/processors/igfs/IgfsEntryInfo.java | 3 -
.../processors/igfs/IgfsMetaManager.java | 60 ++------------------
.../processors/igfs/IgfsModeResolver.java | 2 +-
.../IgfsSecondaryFileSystemCreateContext.java | 13 ++---
.../ignite/testsuites/IgniteIgfsTestSuite.java | 3 +-
.../hadoop/HadoopAbstractMapReduceTest.java | 33 ++++++-----
7 files changed, 34 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
index 8d9a038..d65ad2e 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -361,7 +361,7 @@ public interface IgniteFileSystem extends IgniteAsyncSupport {
* will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
* will be removed from the stored properties or ignored if they don't exist in the file info.
* <p>
- * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be updated:
+ * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes with Hadoop secondary file system only the following properties will be updated:
* <ul>
* <li>{@code usrName} - file owner name;</li>
* <li>{@code grpName} - file owner group;</li>
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java
index e8fc973..4df8b34 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEntryInfo.java
@@ -153,9 +153,6 @@ public abstract class IgfsEntryInfo implements Externalizable {
public IgfsEntryInfo accessModificationTime(long accessTime, long modificationTime) {
IgfsEntryInfo res = copy();
- System.out.println("set: a=" + res.accessTime + " -> " + accessTime);
- System.out.println("set: m=" + res.modificationTime + " -> " + modificationTime);
-
res.accessTime = accessTime;
res.modificationTime = modificationTime;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index b6d7a25..c0f152c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -117,11 +117,8 @@ public class IgfsMetaManager extends IgfsManager {
/** Used for access and modification time for meta cache in dual modes. */
private static final long UNKNOWN_TIME = -1L;
- /** TODO: null? */
- private static final Map<String, String> UNKNOWN_PROPERTIES = Collections.emptyMap();
-
- // TODO: unknown listing
- private static final Map<String, String> UNKNOWN_LISTING = Collections.emptyMap();
+ /** Unknown file properties. */
+ private static final Map<String, String> UNKNOWN_PROPERTIES = null;
/** IGFS configuration. */
private FileSystemConfiguration cfg;
@@ -2234,7 +2231,6 @@ public class IgfsMetaManager extends IgfsManager {
IgfsEntryInfo parentPathInfo = infos.get(parentPath);
- // TODO: remove this call:
synchronize(fs, parentPath, parentPathInfo, path, true, null);
if (evts.isRecordable(EventType.EVT_IGFS_DIR_CREATED)) {
@@ -3061,20 +3057,6 @@ public class IgfsMetaManager extends IgfsManager {
if (secondaryCtx != null) {
secondaryOut = secondaryCtx.create();
-// // NB: this call is unavoidable since we should know the file length:
-// IgfsFile secondaryFile = secondaryCtx.info();
-//
-// if (secondaryFile == null)
-// throw fsException("Failed to open output stream to the file created in " +
-// "the secondary file system because it no longer exists: " + path);
-// else if (secondaryFile.isDirectory())
-// throw fsException("Failed to open output stream to the file created in " +
-// "the secondary file system because the path points to a directory: " + path);
-
-// newAccessTime = secondaryFile.accessTime();
-// newModificationTime = secondaryFile.modificationTime();
-// newProps = secondaryFile.properties();
-
newAccessTime = UNKNOWN_TIME;
newModificationTime = UNKNOWN_TIME;
newProps = UNKNOWN_PROPERTIES;
@@ -3268,16 +3250,6 @@ public class IgfsMetaManager extends IgfsManager {
Map<String, String> props;
if (secondaryCtx != null) {
- // TODO: ***************************
- //IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(lastCreatedPath);
-//
-// if (secondaryInfo == null)
-// throw new IgfsException("Failed to perform operation because secondary file system path was " +
-// "modified concurrently: " + lastCreatedPath);
-// else if (secondaryInfo.isFile())
-// throw new IgfsException("Failed to perform operation because secondary file system entity is " +
-// "not directory: " + lastCreatedPath);
-
accessTime = UNKNOWN_TIME;
modificationTime = UNKNOWN_TIME;
props = UNKNOWN_PROPERTIES;
@@ -3308,19 +3280,6 @@ public class IgfsMetaManager extends IgfsManager {
Map<String, String> props;
if (secondaryCtx != null) {
-// IgfsFile secondaryInfo = secondaryCtx.fileSystem().info(pathIds.path());
-//
-// if (secondaryInfo == null)
-// throw new IgfsException("Failed to perform operation because secondary file system path was " +
-// "modified concurrnetly: " + pathIds.path());
-// else if (secondaryInfo.isFile())
-// throw new IgfsException("Failed to perform operation because secondary file system entity is " +
-// "not directory: " + lastCreatedPath);
-//
-// accessTime = secondaryInfo.accessTime();
-// modificationTime = secondaryInfo.modificationTime();
-// props = secondaryInfo.properties();
-
accessTime = UNKNOWN_TIME;
modificationTime = UNKNOWN_TIME;
props = UNKNOWN_PROPERTIES;
@@ -3341,18 +3300,9 @@ public class IgfsMetaManager extends IgfsManager {
int newBlockSize;
if (secondaryCtx != null) {
-// IgfsFile secondaryFile = secondaryCtx.info();
-//
-// if (secondaryFile == null)
-// throw fsException("Failed to open output stream to the file created in " +
-// "the secondary file system because it no longer exists: " + pathIds.path());
-// else if (secondaryFile.isDirectory())
-// throw fsException("Failed to open output stream to the file created in " +
-// "the secondary file system because the path points to a directory: " + pathIds.path());
-
- newAccessTime = UNKNOWN_TIME; //secondaryFile.accessTime();
- newModificationTime = UNKNOWN_TIME; //secondaryFile.modificationTime();
- newProps = UNKNOWN_PROPERTIES; //secondaryFile.properties();
+ newAccessTime = UNKNOWN_TIME;
+ newModificationTime = UNKNOWN_TIME;
+ newProps = UNKNOWN_PROPERTIES;
}
else {
newAccessTime = curTime;
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
index 3e7df94..907051f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsModeResolver.java
@@ -100,4 +100,4 @@ public class IgfsModeResolver {
@Nullable public List<T2<IgfsPath, IgfsMode>> modesOrdered() {
return modes != null ? Collections.unmodifiableList(modes) : null;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
index 7b2cdcd..788efe3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemCreateContext.java
@@ -100,13 +100,12 @@ public class IgfsSecondaryFileSystemCreateContext {
return fs.info(path);
}
- // TODO: Note used?
-// /**
-// * @return Secondary file system.
-// */
-// public IgfsSecondaryFileSystem fileSystem() {
-// return fs;
-// }
+ /**
+ * @return Secondary file system.
+ */
+ public IgfsSecondaryFileSystem fileSystem() {
+ return fs;
+ }
/** {@inheritDoc} */
@Override public String toString() {
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
index 791672c..dc1379f 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
@@ -124,8 +124,7 @@ public class IgniteIgfsTestSuite extends TestSuite {
suite.addTestSuite(IgfsFixedLengthRecordResolverSelfTest.class);
suite.addTestSuite(IgfsNewLineDelimiterRecordResolverSelfTest.class);
- // TODO: uncomment back:
- // suite.addTestSuite(IgfsTaskSelfTest.class);
+ suite.addTestSuite(IgfsTaskSelfTest.class);
suite.addTestSuite(IgfsGroupDataBlockKeyMapperHashSelfTest.class);
http://git-wip-us.apache.org/repos/asf/ignite/blob/e7dfc178/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
index ef886e4..c0a8d2c 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopAbstractMapReduceTest.java
@@ -41,6 +41,7 @@ import org.apache.ignite.configuration.HadoopConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
+import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsGroupDataBlocksKeyMapper;
import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
import org.apache.ignite.igfs.IgfsMode;
@@ -123,8 +124,16 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
* @param p The path.
* @return The owner.
*/
- private static String getOwner(IgfsEx i, IgfsPath p) {
- return i.info(p).property(IgfsUtils.PROP_USER_NAME);
+ private static String getOwner(final IgfsEx i, final IgfsPath p) {
+ return IgfsUserContext.doAs(USER, new IgniteOutClosure<String>() {
+ @Override public String apply() {
+ IgfsFile f = i.info(p);
+
+ assert f != null;
+
+ return f.property(IgfsUtils.PROP_USER_NAME);
+ }
+ });
}
/**
@@ -160,8 +169,8 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
* @param useNewCombiner flag to use new combiner API.
* @param useNewReducer flag to use new reducer API.
*/
- protected final void doTest(IgfsPath inFile, boolean useNewMapper, boolean useNewCombiner, boolean useNewReducer)
- throws Exception {
+ protected final void doTest(IgfsPath inFile, boolean useNewMapper, boolean useNewCombiner,
+ boolean useNewReducer) throws Exception {
igfs.delete(new IgfsPath(PATH_OUTPUT), true);
JobConf jobConf = new JobConf();
@@ -210,13 +219,10 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
String actual = readAndSortFile(outFile, job.getConfiguration());
assertEquals("Use new mapper: " + useNewMapper + ", new combiner: " + useNewCombiner + ", new reducer: " +
- useNewReducer,
- "blue\t" + blue + "\n" +
+ useNewReducer, "blue\t" + blue + "\n" +
"green\t" + green + "\n" +
"red\t" + red + "\n" +
- "yellow\t" + yellow + "\n",
- actual
- );
+ "yellow\t" + yellow + "\n", actual);
}
/**
@@ -239,7 +245,7 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
HadoopPerformanceCounter perfCntr = HadoopPerformanceCounter.getCounter(cntrs, null);
- Map<String, SortedMap<Integer,Long>> tasks = new TreeMap<>();
+ Map<String, SortedMap<Integer, Long>> tasks = new TreeMap<>();
Map<String, Integer> phaseOrders = new HashMap<>();
phaseOrders.put("submit", 0);
@@ -269,7 +275,7 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
}
if (!taskId.equals(prevTaskId))
- tasks.put(taskId, new TreeMap<Integer,Long>());
+ tasks.put(taskId, new TreeMap<Integer, Long>());
Integer pos = phaseOrders.get(taskPhase);
@@ -282,7 +288,7 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
apiEvtCnt++;
}
- for (Map.Entry<String ,SortedMap<Integer,Long>> task : tasks.entrySet()) {
+ for (Map.Entry<String, SortedMap<Integer, Long>> task : tasks.entrySet()) {
Map<Integer, Long> order = task.getValue();
long prev = 0;
@@ -344,7 +350,8 @@ public class HadoopAbstractMapReduceTest extends HadoopAbstractWordCountTest {
* @throws Exception If failed.
*/
protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
+ @Nullable IgfsSecondaryFileSystem secondaryFs,
+ @Nullable IgfsIpcEndpointConfiguration restCfg) throws Exception {
FileSystemConfiguration igfsCfg = new FileSystemConfiguration();
igfsCfg.setDataCacheName("dataCache");