You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2015/12/23 11:40:46 UTC
[04/12] ignite git commit: 2206: intermediate commit for review.
2206: intermediate commit for review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d485969f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d485969f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d485969f
Branch: refs/heads/ignite-2206
Commit: d485969fbf8e95017e0601f1f0c35d31043599a7
Parents: 3b03b44
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 16:44:47 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 16:44:47 2015 +0300
----------------------------------------------------------------------
.../ignite/igfs/HadoopFileSystemFactory.java | 19 ++--
.../processors/igfs/IgfsHandshakeResponse.java | 19 +++-
.../internal/processors/igfs/IgfsImpl.java | 2 +-
.../internal/processors/igfs/IgfsPaths.java | 102 +++++++++----------
.../visor/node/VisorIgfsConfiguration.java | 38 +++----
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 31 +++++-
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 13 ++-
.../fs/DefaultHadoopFileSystemFactory.java | 1 -
8 files changed, 137 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
index 86d39e1..6720da9 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
@@ -1,15 +1,14 @@
package org.apache.ignite.igfs;
-import java.io.Externalizable;
import java.io.IOException;
-import java.net.URI;
+import java.io.Serializable;
/**
- * This factory is {@link Externalizable} because it should be transferable over the network.
+ * This factory is {@link Serializable} because it should be transferable over the network.
*
* @param <T> The type
*/
-public interface HadoopFileSystemFactory <T> extends Externalizable {
+public interface HadoopFileSystemFactory <T> extends Serializable {
/**
* Gets the file system, possibly creating it or taking a cached instance.
* All the other data needed for the file system creation are expected to be contained
@@ -21,10 +20,10 @@ public interface HadoopFileSystemFactory <T> extends Externalizable {
*/
public T get(String userName) throws IOException;
- /**
- * Getter for the file system URI.
- *
- * @return The file system URI.
- */
- public URI uri();
+// /**
+// * Getter for the file system URI.
+// *
+// * @return The file system URI.
+// */
+// public URI uri();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
index 1ba98ac..230f798 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
@@ -36,6 +36,9 @@ public class IgfsHandshakeResponse implements Externalizable {
/** SECONDARY paths. */
private IgfsPaths paths;
+ /** */
+ private byte[] secondaryFileSystemFactoryBytes;
+
/** Server block size. */
private long blockSize;
@@ -55,10 +58,12 @@ public class IgfsHandshakeResponse implements Externalizable {
* @param paths Secondary paths.
* @param blockSize Server default block size.
*/
- public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, long blockSize, Boolean sampling) {
+ public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, byte[] secondaryFileSystemFactoryBytes,
+ long blockSize, Boolean sampling) {
assert paths != null;
this.igfsName = igfsName;
+ this.secondaryFileSystemFactoryBytes = secondaryFileSystemFactoryBytes;
this.paths = paths;
this.blockSize = blockSize;
this.sampling = sampling;
@@ -92,12 +97,22 @@ public class IgfsHandshakeResponse implements Externalizable {
return sampling;
}
+ /**
+ *
+ * @return
+ */
+ public byte[] getSecondaryFileSystemFactoryBytes() {
+ return secondaryFileSystemFactoryBytes;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeString(out, igfsName);
paths.writeExternal(out);
+ U.writeByteArray(out, secondaryFileSystemFactoryBytes);
+
out.writeLong(blockSize);
if (sampling != null) {
@@ -116,6 +131,8 @@ public class IgfsHandshakeResponse implements Externalizable {
paths.readExternal(in);
+ secondaryFileSystemFactoryBytes = U.readByteArray(in);
+
blockSize = in.readLong();
if (in.readBoolean())
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 7ea0333..15509ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -256,7 +256,7 @@ public final class IgfsImpl implements IgfsEx {
secondaryPaths = new IgfsPaths(
secondaryFs == null ? null : secondaryFs.properties(),
- secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
+ //secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
dfltMode,
modeRslvr.modesOrdered());
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index 809c7da..a9813d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -28,7 +28,7 @@ import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
+//import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.typedef.T2;
@@ -46,8 +46,8 @@ public class IgfsPaths <F> implements Externalizable {
@Deprecated
private Map<String, String> props;
- /** */
- private HadoopFileSystemFactory<F> factory;
+// /** */
+// private HadoopFileSystemFactory<F> factory;
/** Default IGFS mode. */
private IgfsMode dfltMode;
@@ -70,11 +70,11 @@ public class IgfsPaths <F> implements Externalizable {
* @param pathModes Path modes.
*/
public IgfsPaths(Map<String, String> props,
- HadoopFileSystemFactory<F> factory,
+ //HadoopFileSystemFactory<F> factory,
IgfsMode dfltMode,
@Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
this.props = props;
- this.factory = factory;
+ //this.factory = factory;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
}
@@ -103,20 +103,20 @@ public class IgfsPaths <F> implements Externalizable {
return pathModes;
}
- /**
- * Getter for factory.
- *
- * @return The factory.
- */
- public HadoopFileSystemFactory<F> factory() {
- return factory;
- }
+// /**
+// * Getter for factory.
+// *
+// * @return The factory.
+// */
+// public HadoopFileSystemFactory<F> factory() {
+// return factory;
+// }
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeStringMap(out, props);
- writeFactory(out);
+// writeFactory(out);
U.writeEnum(out, dfltMode);
@@ -133,30 +133,30 @@ public class IgfsPaths <F> implements Externalizable {
out.writeBoolean(false);
}
- /**
- *
- * @param out
- * @throws IOException
- */
- private void writeFactory(ObjectOutput out) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- ObjectOutput oo = new ObjectOutputStream(baos);
- try {
- oo.writeObject(factory);
- }
- finally {
- oo.close();
- }
-
- U.writeByteArray(out, baos.toByteArray());
- }
+// /**
+// *
+// * @param out
+// * @throws IOException
+// */
+// private void writeFactory(ObjectOutput out) throws IOException {
+// ByteArrayOutputStream baos = new ByteArrayOutputStream();
+//
+// ObjectOutput oo = new ObjectOutputStream(baos);
+// try {
+// oo.writeObject(factory);
+// }
+// finally {
+// oo.close();
+// }
+//
+// U.writeByteArray(out, baos.toByteArray());
+// }
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
props = U.readStringMap(in);
- readFactory(in);
+// readFactory(in);
dfltMode = IgfsMode.fromOrdinal(in.readByte());
@@ -176,22 +176,22 @@ public class IgfsPaths <F> implements Externalizable {
}
}
- /**
- *
- * @param in
- * @throws IOException
- * @throws ClassNotFoundException
- */
- private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
- byte[] factoryBytes = U.readByteArray(in);
-
- ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
-
- try {
- factory = (HadoopFileSystemFactory<F>) oi.readObject();
- }
- finally {
- oi.close();
- }
- }
+// /**
+// *
+// * @param in
+// * @throws IOException
+// * @throws ClassNotFoundException
+// */
+// private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
+// byte[] factoryBytes = U.readByteArray(in);
+//
+// ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+//
+// try {
+// factory = (HadoopFileSystemFactory<F>) oi.readObject();
+// }
+// finally {
+// oi.close();
+// }
+// }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 4a2e7b1..08792a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -75,7 +75,7 @@ public class VisorIgfsConfiguration implements Serializable {
// /** User name for the secondary hadoop file system config. */
// private String secondaryHadoopFileSysUserName;
- private HadoopFileSystemFactory factory;
+// private HadoopFileSystemFactory factory;
/** IGFS instance mode. */
private IgfsMode dfltMode;
@@ -146,16 +146,16 @@ public class VisorIgfsConfiguration implements Serializable {
IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
- if (secFs != null) {
- //Map<String, String> props = secFs.properties();
-
- //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
- //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
- //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
-
- // Just take and save the factory object:
- cfg.factory = secFs.getSecondaryFileSystemFactory();
- }
+// if (secFs != null) {
+// //Map<String, String> props = secFs.properties();
+//
+// //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
+// //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
+// //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
+//
+// // Just take and save the factory object:
+// cfg.factory = secFs.getSecondaryFileSystemFactory();
+// }
cfg.dfltMode = igfs.getDefaultMode();
cfg.pathModes = igfs.getPathModes();
@@ -277,14 +277,14 @@ public class VisorIgfsConfiguration implements Serializable {
// return secondaryHadoopFileSysCfgPath;
// }
- /**
- *
- * @param <T>
- * @return
- */
- public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
- return factory;
- }
+// /**
+// *
+// * @param <T>
+// * @return
+// */
+// public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
+// return factory;
+// }
/**
* @return IGFS instance mode.
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 2fcf774..20e2011 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -18,9 +18,12 @@
package org.apache.ignite.hadoop.fs.v1;
import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
@@ -331,7 +334,11 @@ public class IgniteHadoopFileSystem extends FileSystem {
// String secUri = props.get(SECONDARY_FS_URI);
// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- HadoopFileSystemFactory<FileSystem> factory = paths.factory();
+ byte[] secFsFacoryBytes = handshake.getSecondaryFileSystemFactoryBytes();
+
+
+
+ //HadoopFileSystemFactory<FileSystem> factory = paths.factory();
A.ensure(factory != null, "Secondary file system factory should not be null.");
@@ -343,6 +350,10 @@ public class IgniteHadoopFileSystem extends FileSystem {
//SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
secondaryFs = factory.get(user); //secProvider.createFileSystem(user);
+
+ URI uri2 = secondaryFs.getUri();
+
+ assert secondaryUri.equals(uri2);
}
catch (IOException e) {
if (!mgmt)
@@ -361,6 +372,24 @@ public class IgniteHadoopFileSystem extends FileSystem {
}
}
+ /**
+ *
+ * @param in
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ static HadoopFileSystemFactory readFactory(byte[] factoryBytes) throws IOException, ClassNotFoundException {
+ ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+
+ try {
+ return (HadoopFileSystemFactory<F>) oi.readObject();
+ }
+ finally {
+ oi.close();
+ }
+ }
+
+
/** {@inheritDoc} */
@Override protected void checkPath(Path path) {
URI uri = path.toUri();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index 865a2bc..ae97464 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -58,6 +58,7 @@ import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.igfs.common.IgfsLogger;
+import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutputStream;
@@ -345,17 +346,21 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
A.ensure(secondaryUri != null, "File system factory uri should not be null.");
- secondaryUri = factory.uri();
-
- A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
+ //secondaryUri = factory.uri();
try {
//SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
secondaryFs = factory.get(user);
- //secondaryFs = secProvider.createAbstractFileSystem(user);
+ secondaryUri = secondaryFs.getUri();
+
+ assert secondaryUri != null;
+ URI uri2 = ((DefaultHadoopFileSystemFactory)factory).uri();
+ assert secondaryUri.equals(uri2);
+
+ //secondaryFs = secProvider.createAbstractFileSystem(user);
//secondaryUri = secProvider.uri();
}
catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
index 246637d..0b93cba 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
@@ -106,7 +106,6 @@ public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<F
}
}
-
protected void init() throws IOException {
String secUri = nullifyEmpty(uri == null ? null : uri.toString());