You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2015/12/23 11:40:43 UTC
[01/12] ignite git commit: 2206
Repository: ignite
Updated Branches:
refs/heads/ignite-2206 [created] 31d3289df
2206
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/7a3f9dd3
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/7a3f9dd3
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/7a3f9dd3
Branch: refs/heads/ignite-2206
Commit: 7a3f9dd35ccf487e64318c8bf76925ad0bac0339
Parents: 2848680
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Dec 21 17:41:20 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Dec 21 17:41:20 2015 +0300
----------------------------------------------------------------------
.../ignite/igfs/HadoopFileSystemFactory.java | 30 +++
.../igfs/secondary/IgfsSecondaryFileSystem.java | 11 +-
.../ignite/internal/processors/igfs/IgfsEx.java | 24 +--
.../internal/processors/igfs/IgfsImpl.java | 5 +-
.../internal/processors/igfs/IgfsPaths.java | 32 ++-
.../igfs/IgfsSecondaryFileSystemImpl.java | 8 +-
.../ignite/internal/util/lang/GridFunc.java | 24 +++
.../visor/node/VisorIgfsConfiguration.java | 75 ++++---
.../org/apache/ignite/hadoop/HadoopFsIssue.java | 71 +++++++
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 207 ++++++++++++-------
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 28 +--
.../hadoop/fs/v2/HadoopV2FileSystemFactory.java | 11 +
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 33 ++-
.../hadoop/IgfsSecondaryFileSystemEx.java | 15 ++
.../KerberosSecondaryFileSystemProvider.java | 55 +++++
.../hadoop/SecondaryFileSystemProvider.java | 29 ++-
.../fs/DefaultHadoopFileSystemFactory.java | 183 ++++++++++++++++
...oopFileSystemUniversalFileSystemAdapter.java | 2 +
.../testsuites/IgniteHadoopTestSuite.java | 2 +-
parent/pom.xml | 2 +-
20 files changed, 696 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
new file mode 100644
index 0000000..86d39e1
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
@@ -0,0 +1,30 @@
+package org.apache.ignite.igfs;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * This factory is {@link Externalizable} because it should be transferable over the network.
+ *
+ * @param <T> The type
+ */
+public interface HadoopFileSystemFactory <T> extends Externalizable {
+ /**
+ * Gets the file system, possibly creating it or taking a cached instance.
+ * All the other data needed for the file system creation are expected to be contained
+ * in this object instance.
+ *
+ * @param userName The user name
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public T get(String userName) throws IOException;
+
+ /**
+ * Getter for the file system URI.
+ *
+ * @return The file system URI.
+ */
+ public URI uri();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index ca6ecb7..696c81a 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -21,6 +21,7 @@ import java.io.OutputStream;
import java.util.Collection;
import java.util.Map;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.jetbrains.annotations.Nullable;
@@ -28,7 +29,7 @@ import org.jetbrains.annotations.Nullable;
/**
* Secondary file system interface.
*/
-public interface IgfsSecondaryFileSystem {
+public interface IgfsSecondaryFileSystem <F> {
/**
* Checks if the specified path exists.
*
@@ -197,9 +198,17 @@ public interface IgfsSecondaryFileSystem {
* Gets the implementation specific properties of file system.
*
* @return Map of properties.
+ * @deprecated Should not be used.
*/
+ @Deprecated
public Map<String,String> properties();
+ /**
+ *
+ * @return The factory.
+ */
+ public @Nullable HadoopFileSystemFactory<F> getSecondaryFileSystemFactory();
+
/**
* Closes the secondary file system.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index 8ff7247..a338813 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -43,18 +43,18 @@ public interface IgfsEx extends IgniteFileSystem {
/** File property: prefer writes to local node. */
public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
- /** Property name for path to Hadoop configuration. */
- public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
-
- /** Property name for URI of file system. */
- public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
-
- /** Property name for default user name of file system.
- * NOTE: for secondary file system this is just a default user name, which is used
- * when the 2ndary filesystem is used outside of any user context.
- * If another user name is set in the context, 2ndary file system will work on behalf
- * of that user, which is different from the default. */
- public static final String SECONDARY_FS_USER_NAME = "SECONDARY_FS_USER_NAME";
+// /** Property name for path to Hadoop configuration. */
+// public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
+//
+// /** Property name for URI of file system. */
+// public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
+//
+// /** Property name for default user name of file system.
+// * NOTE: for secondary file system this is just a default user name, which is used
+// * when the 2ndary filesystem is used outside of any user context.
+// * If another user name is set in the context, 2ndary file system will work on behalf
+// * of that user, which is different from the default. */
+// public static final String SECONDARY_FS_USER_NAME = "SECONDARY_FS_USER_NAME";
/**
* Stops IGFS cleaning all used resources.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 0d5cda3..7ea0333 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -254,7 +254,10 @@ public final class IgfsImpl implements IgfsEx {
modeRslvr = new IgfsModeResolver(dfltMode, modes);
- secondaryPaths = new IgfsPaths(secondaryFs == null ? null : secondaryFs.properties(), dfltMode,
+ secondaryPaths = new IgfsPaths(
+ secondaryFs == null ? null : secondaryFs.properties(),
+ secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
+ dfltMode,
modeRslvr.modesOrdered());
// Check whether IGFS LRU eviction policy is set on data cache.
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index fbf89ce..bf7e825 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -24,6 +24,7 @@ import java.io.ObjectOutput;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.typedef.T2;
@@ -33,13 +34,17 @@ import org.jetbrains.annotations.Nullable;
/**
* Description of path modes.
*/
-public class IgfsPaths implements Externalizable {
+public class IgfsPaths <F> implements Externalizable {
/** */
private static final long serialVersionUID = 0L;
/** Additional secondary file system properties. */
+ @Deprecated
private Map<String, String> props;
+ /** */
+ private HadoopFileSystemFactory<F> factory;
+
/** Default IGFS mode. */
private IgfsMode dfltMode;
@@ -60,16 +65,22 @@ public class IgfsPaths implements Externalizable {
* @param dfltMode Default IGFS mode.
* @param pathModes Path modes.
*/
- public IgfsPaths(Map<String, String> props, IgfsMode dfltMode, @Nullable List<T2<IgfsPath,
- IgfsMode>> pathModes) {
+ public IgfsPaths(Map<String, String> props,
+ HadoopFileSystemFactory<F> factory,
+ IgfsMode dfltMode,
+ @Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
this.props = props;
+ this.factory = factory;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
}
/**
* @return Secondary file system properties.
+ *
+ * @deprecated
*/
+ @Deprecated
public Map<String, String> properties() {
return props;
}
@@ -88,9 +99,21 @@ public class IgfsPaths implements Externalizable {
return pathModes;
}
+ /**
+ * Getter for factory.
+ *
+ * @return The factory.
+ */
+ public HadoopFileSystemFactory<F> factory() {
+ return factory;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeStringMap(out, props);
+
+ out.writeObject(factory);
+
U.writeEnum(out, dfltMode);
if (pathModes != null) {
@@ -109,6 +132,9 @@ public class IgfsPaths implements Externalizable {
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
props = U.readStringMap(in);
+
+ factory = (HadoopFileSystemFactory<F>)in.readObject();
+
dfltMode = IgfsMode.fromOrdinal(in.readByte());
if (in.readBoolean()) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
index 23d6322..1b1ce24 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
@@ -22,6 +22,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
@@ -31,7 +32,7 @@ import org.jetbrains.annotations.Nullable;
/**
* Secondary file system over native IGFS.
*/
-class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
+class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem <IgfsEx> {
/** Delegate. */
private final IgfsEx igfs;
@@ -126,4 +127,9 @@ class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
@Override public void close() throws IgniteException {
// No-op.
}
+
+ /** {@inheritDoc} */
+ @Override public HadoopFileSystemFactory<IgfsEx> getSecondaryFileSystemFactory() {
+ return null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 8d5a8e7..224d205 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -2290,6 +2290,30 @@ public class GridFunc {
}
/**
+ * Nullifies an empty String.
+ * @param x The argument.
+ * @return Nullified argument.
+ */
+ public static String nullifyEmpty(String x) {
+ if (isEmpty(x))
+ return null;
+
+ return x;
+ }
+
+ /**
+ * Nullifies an empty collection.
+ * @param c The argument.
+ * @return Nullified argument.
+ */
+ public static <T> Collection<T> nullifyEmpty(Collection<T> c) {
+ if (isEmpty(c))
+ return null;
+
+ return c;
+ }
+
+ /**
* Utility map getter. This method analogous to {@link #addIfAbsent(Map, Object, Callable)}
* method but this one doesn't put the default value into the map when key is not found.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index e85484d..4a2e7b1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -23,15 +23,16 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import org.apache.ignite.configuration.FileSystemConfiguration;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_USER_NAME;
+//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
+//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
+//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_USER_NAME;
import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactClass;
/**
@@ -65,14 +66,16 @@ public class VisorIgfsConfiguration implements Serializable {
/** Number of batches that can be concurrently sent to remote node. */
private int perNodeParallelBatchCnt;
- /** URI of the secondary Hadoop file system. */
- private String secondaryHadoopFileSysUri;
+// /** URI of the secondary Hadoop file system. */
+// private String secondaryHadoopFileSysUri;
+//
+// /** Path for the secondary hadoop file system config. */
+// private String secondaryHadoopFileSysCfgPath;
+//
+// /** User name for the secondary hadoop file system config. */
+// private String secondaryHadoopFileSysUserName;
- /** Path for the secondary hadoop file system config. */
- private String secondaryHadoopFileSysCfgPath;
-
- /** User name for the secondary hadoop file system config. */
- private String secondaryHadoopFileSysUserName;
+ private HadoopFileSystemFactory factory;
/** IGFS instance mode. */
private IgfsMode dfltMode;
@@ -144,11 +147,14 @@ public class VisorIgfsConfiguration implements Serializable {
IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
if (secFs != null) {
- Map<String, String> props = secFs.properties();
+ //Map<String, String> props = secFs.properties();
+
+ //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
+ //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
+ //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
- cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
- cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
- cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
+ // Just take and save the factory object:
+ cfg.factory = secFs.getSecondaryFileSystemFactory();
}
cfg.dfltMode = igfs.getDefaultMode();
@@ -250,25 +256,34 @@ public class VisorIgfsConfiguration implements Serializable {
return perNodeParallelBatchCnt;
}
- /**
- * @return URI of the secondary Hadoop file system.
- */
- @Nullable public String secondaryHadoopFileSystemUri() {
- return secondaryHadoopFileSysUri;
- }
-
- /**
- * @return User name of the secondary Hadoop file system.
- */
- @Nullable public String secondaryHadoopFileSystemUserName() {
- return secondaryHadoopFileSysUserName;
- }
+// /**
+// * @return URI of the secondary Hadoop file system.
+// */
+// @Nullable public String secondaryHadoopFileSystemUri() {
+// return secondaryHadoopFileSysUri;
+// }
+//
+// /**
+// * @return User name of the secondary Hadoop file system.
+// */
+// @Nullable public String secondaryHadoopFileSystemUserName() {
+// return secondaryHadoopFileSysUserName;
+// }
+//
+// /**
+// * @return Path for the secondary hadoop file system config.
+// */
+// @Nullable public String secondaryHadoopFileSystemConfigPath() {
+// return secondaryHadoopFileSysCfgPath;
+// }
/**
- * @return Path for the secondary hadoop file system config.
+ *
+ * @param <T>
+ * @return
*/
- @Nullable public String secondaryHadoopFileSystemConfigPath() {
- return secondaryHadoopFileSysCfgPath;
+ public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
+ return factory;
}
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/HadoopFsIssue.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/HadoopFsIssue.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/HadoopFsIssue.java
new file mode 100644
index 0000000..82314f1
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/HadoopFsIssue.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop;
+
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider;
+
+/**
+ * Comment.
+ */
+public class HadoopFsIssue {
+ /**
+ *
+ * @param args
+ */
+ public static void main(String args[]) {
+ String uri = null;
+ String cfgPath = null;
+ String user = null;
+
+ for (String arg : args) {
+ if (arg.startsWith("uri="))
+ uri = arg.split("=")[1].trim();
+ else if (arg.startsWith("cfg="))
+ cfgPath = arg.split("=")[1].trim();
+ else if (arg.startsWith("user="))
+ user = arg.split("=")[1].trim();
+ else
+ throw new IllegalArgumentException("Unknown argument:" + arg);
+ }
+
+ System.out.println("Connecting to HDFS with the following settings [uri=" + uri + ", cfg=" + cfgPath + ", user=" + user + ']');
+
+ try {
+ SecondaryFileSystemProvider provider = new SecondaryFileSystemProvider(uri, cfgPath);
+
+ FileSystem fs = provider.createFileSystem(user);
+
+ RemoteIterator<LocatedFileStatus> iter = fs.listFiles(new Path("/tmp"), true);
+
+ System.out.println("Got the iterator");
+
+ while (iter.hasNext()) {
+ LocatedFileStatus status = iter.next();
+
+ System.out.println(status);
+ }
+ }
+ catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 1ca6938..c5124f0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -24,7 +24,6 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.HashMap;
import java.util.Map;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -35,6 +34,7 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -45,64 +45,80 @@ import org.apache.ignite.igfs.IgfsPathNotFoundException;
import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
-import org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider;
-import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
-import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap.ValueFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable;
-import org.apache.ignite.internal.processors.igfs.IgfsEx;
import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
import org.apache.ignite.internal.processors.igfs.IgfsFileInfo;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.F;
+import static org.apache.ignite.internal.util.typedef.F.*;
+
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_USER_NAME;
/**
* Adapter to use any Hadoop file system {@link FileSystem} as {@link IgfsSecondaryFileSystem}.
* In fact, this class deals with different FileSystems depending on the user context,
* see {@link IgfsUserContext#currentUser()}.
*/
-public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem {
- /** Properties of file system, see {@link #properties()}
- *
- * See {@link IgfsEx#SECONDARY_FS_CONFIG_PATH}
- * See {@link IgfsEx#SECONDARY_FS_URI}
- * See {@link IgfsEx#SECONDARY_FS_USER_NAME}
- * */
- private final Map<String, String> props = new HashMap<>();
+public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem <FileSystem> {
+// /** Properties of file system, see {@link #properties()}
+// * */
+// private final Map<String, String> props = new HashMap<>();
/** Secondary file system provider. */
- private final SecondaryFileSystemProvider secProvider;
-
- /** The default user name. It is used if no user context is set. */
- private final String dfltUserName;
+ //private SecondaryFileSystemProvider secProvider;
/** FileSystem instance created for the default user.
* Stored outside the fileSysLazyMap due to performance reasons. */
- private final FileSystem dfltFs;
+ private FileSystem dfltFs;
+
+// /** */
+// private String uriStr;
+//
+// /** Note: */
+// private URI uri;
+//
+// /** */
+// private Collection<String> cfgPathsStr;
+//
+// /** */
+// private @Nullable Collection<URI> cfgPaths;
- /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
- private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
- new ValueFactory<String, FileSystem>() {
- @Override public FileSystem createValue(String key) {
- try {
- assert !F.isEmpty(key);
+ /** The default user name. It is used if no user context is set. */
+ private String dfltUserName = IgfsUtils.fixUserName(null);
+
+ /** */
+ private HadoopFileSystemFactory<FileSystem> fsFactory;
+
+// /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
+// private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
+// new ValueFactory<String, FileSystem>() {
+// @Override public FileSystem createValue(String key) {
+// try {
+// assert !F.isEmpty(key);
+//
+// return secProvider.createFileSystem(key);
+// }
+// catch (IOException ioe) {
+// throw new IgniteException(ioe);
+// }
+// }
+// }
+// );
- return secProvider.createFileSystem(key);
- }
- catch (IOException ioe) {
- throw new IgniteException(ioe);
- }
- }
- }
- );
+ /**
+ * Default constructor for Spring.
+ */
+ public IgniteHadoopIgfsSecondaryFileSystem() {
+ // noop.
+ }
/**
* Simple constructor that is to be used by default.
@@ -136,43 +152,54 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
*/
public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath,
@Nullable String userName) throws IgniteCheckedException {
- // Treat empty uri and userName arguments as nulls to improve configuration usability:
- if (F.isEmpty(uri))
- uri = null;
-
- if (F.isEmpty(cfgPath))
- cfgPath = null;
-
- if (F.isEmpty(userName))
- userName = null;
- this.dfltUserName = IgfsUtils.fixUserName(userName);
+ uri = nullifyEmpty(uri);
+ if (uri != null)
+ U.warn(null, "This constructor is deprecated. URI value passed in will be ignored.");
- try {
- this.secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
-
- // File system creation for the default user name.
- // The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
- this.dfltFs = secProvider.createFileSystem(dfltUserName);
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
-
- assert dfltFs != null;
+ cfgPath = nullifyEmpty(cfgPath);
+ if (cfgPath != null)
+ U.warn(null, "This constructor is deprecated. The configurationPath value passed in will be ignored.");
- uri = secProvider.uri().toString();
+ setDfltUserName(userName);
+ }
- if (!uri.endsWith("/"))
- uri += "/";
+ /**
+ *
+ * @param factory
+ */
+ public void setFsFactory(HadoopFileSystemFactory factory) {
+ A.ensure(factory != null, "Factory value must not be null.");
- if (cfgPath != null)
- props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
+ this.fsFactory = factory;
+ }
- props.put(SECONDARY_FS_URI, uri);
- props.put(SECONDARY_FS_USER_NAME, dfltUserName);
+ /**
+ *
+ * @param dfltUserName
+ */
+ public void setDfltUserName(String dfltUserName) {
+ this.dfltUserName = IgfsUtils.fixUserName(nullifyEmpty(dfltUserName));
}
+// /**
+// * Sets the file system properties.
+// */
+// private void setProperties() {
+// String uri = this.uri.toString();
+//
+// if (!uri.endsWith("/"))
+// uri += "/";
+//
+// String cfgPath = secProvider.configurationPath();
+//
+// if (cfgPath != null)
+// props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
+//
+// props.put(SECONDARY_FS_URI, uri);
+// props.put(SECONDARY_FS_USER_NAME, dfltUserName);
+// }
+
/**
* Convert IGFS path into Hadoop path.
*
@@ -488,7 +515,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public Map<String, String> properties() {
- return props;
+ return Collections.emptyMap();
}
/** {@inheritDoc} */
@@ -496,18 +523,20 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
Exception e = null;
try {
- dfltFs.close();
+ if (dfltFs != null)
+ dfltFs.close();
}
catch (Exception e0) {
e = e0;
}
try {
- fileSysLazyMap.close();
+ if (fsFactory instanceof LifecycleAware)
+ ((LifecycleAware)fsFactory).stop();
}
- catch (IgniteCheckedException ice) {
+ catch (IgniteException ie) {
if (e == null)
- e = ice;
+ e = ie;
}
if (e != null)
@@ -538,6 +567,44 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
if (F.eq(user, dfltUserName))
return dfltFs; // optimization
- return fileSysLazyMap.getOrCreate(user);
+ try {
+ return fsFactory.get(user);
+ }
+ catch (IOException ioe) {
+ throw new IgniteException(ioe);
+ }
+ }
+
+ /**
+ * Should be invoked by client (from Spring?) after all the setters invoked.
+ * TODO: how this should be invoked?
+ *
+ * @throws IgniteCheckedException
+ */
+ public void start() throws IgniteCheckedException {
+ A.ensure(fsFactory != null, "factory");
+ A.ensure(dfltUserName != null, "dfltUserName");
+
+ if (fsFactory instanceof LifecycleAware)
+ ((LifecycleAware) fsFactory).start();
+
+ try {
+ //this.secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
+
+ // File system creation for the default user name.
+ // The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
+ //this.dfltFs = secProvider.createFileSystem(dfltUserName);
+ this.dfltFs = fsFactory.get(dfltUserName);
+
+ assert dfltFs != null;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public HadoopFileSystemFactory<FileSystem> getSecondaryFileSystemFactory() {
+ return fsFactory;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 778792a..1b748fb 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -51,7 +52,6 @@ import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.igfs.IgfsPathSummary;
import org.apache.ignite.internal.igfs.common.IgfsLogger;
-import org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProxyInputStream;
@@ -85,8 +85,6 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PREFER_LOCAL_WRITES;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
/**
* {@code IGFS} Hadoop 1.x file system driver over file system API. To use
@@ -293,7 +291,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
igfsGrpBlockSize = handshake.blockSize();
- IgfsPaths paths = handshake.secondaryPaths();
+ final IgfsPaths<FileSystem> paths = handshake.secondaryPaths();
// Initialize client logger.
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -327,21 +325,27 @@ public class IgniteHadoopFileSystem extends FileSystem {
}
if (initSecondary) {
- Map<String, String> props = paths.properties();
+// Map<String, String> props = paths.properties();
+//
+// String secUri = props.get(SECONDARY_FS_URI);
+// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- String secUri = props.get(SECONDARY_FS_URI);
- String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
+ HadoopFileSystemFactory<FileSystem> factory = paths.factory();
- try {
- SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
+ A.ensure(factory != null, "Secondary file system factory should not be null.");
+
+ secondaryUri = factory.uri();
- secondaryFs = secProvider.createFileSystem(user);
+ A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
+
+ try {
+ //SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
- secondaryUri = secProvider.uri();
+ secondaryFs = factory.get(user); //secProvider.createFileSystem(user);
}
catch (IOException e) {
if (!mgmt)
- throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
+ throw new IOException("Failed to connect to the secondary file system: " + secondaryUri, e);
else
LOG.warn("Visor failed to create secondary file system (operations on paths with PROXY mode " +
"will have no effect): " + e.getMessage());
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopV2FileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopV2FileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopV2FileSystemFactory.java
new file mode 100644
index 0000000..c2ab620
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopV2FileSystemFactory.java
@@ -0,0 +1,11 @@
+//package org.apache.ignite.hadoop.fs.v2;
+//
+//import org.apache.hadoop.fs.AbstractFileSystem;
+//import org.apache.hadoop.fs.FileSystem;
+//
+///**
+// * Created by ivan on 18.12.15.
+// */
+//public interface HadoopV2FileSystemFactory {
+// AbstractFileSystem create(String uri, String configPath, String userName);
+//}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index 99ca1ec..865a2bc 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -52,12 +52,12 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.igfs.common.IgfsLogger;
-import org.apache.ignite.internal.processors.hadoop.SecondaryFileSystemProvider;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutputStream;
@@ -92,8 +92,8 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PREFER_LOCAL_WRITES;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
+//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
+//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
/**
* {@code IGFS} Hadoop 2.x file system driver over file system API. To use
@@ -302,7 +302,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
grpBlockSize = handshake.blockSize();
- IgfsPaths paths = handshake.secondaryPaths();
+ IgfsPaths<AbstractFileSystem> paths = handshake.secondaryPaths();
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -335,20 +335,31 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
}
if (initSecondary) {
- Map<String, String> props = paths.properties();
+// Map<String, String> props = paths.properties();
+//
+// String secUri = props.get(SECONDARY_FS_URI);
+// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- String secUri = props.get(SECONDARY_FS_URI);
- String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
+ HadoopFileSystemFactory<AbstractFileSystem> factory
+ = (HadoopFileSystemFactory<AbstractFileSystem>)paths.factory();
+
+ A.ensure(secondaryUri != null, "File system factory uri should not be null.");
+
+ secondaryUri = factory.uri();
+
+ A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
try {
- SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
+ //SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
+
+ secondaryFs = factory.get(user);
- secondaryFs = secProvider.createAbstractFileSystem(user);
+ //secondaryFs = secProvider.createAbstractFileSystem(user);
- secondaryUri = secProvider.uri();
+ //secondaryUri = secProvider.uri();
}
catch (IOException e) {
- throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
+ throw new IOException("Failed to connect to the secondary file system: " + secondaryUri, e);
}
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
new file mode 100644
index 0000000..582e798
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
@@ -0,0 +1,15 @@
+//package org.apache.ignite.internal.processors.hadoop;
+//
+//import org.apache.ignite.IgniteCheckedException;
+//import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
+//
+///**
+// *
+// */
+//public interface IgfsSecondaryFileSystemEx extends IgfsSecondaryFileSystem {
+// /**
+// *
+// * @throws IgniteCheckedException
+// */
+// public void start() throws IgniteCheckedException;
+//}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/KerberosSecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/KerberosSecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/KerberosSecondaryFileSystemProvider.java
new file mode 100644
index 0000000..503ac46
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/KerberosSecondaryFileSystemProvider.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.jetbrains.annotations.Nullable;
+
+/**
+ * See https://issues.apache.org/jira/browse/IGNITE-2195 .
+ */
+public class KerberosSecondaryFileSystemProvider extends SecondaryFileSystemProvider {
+ /**
+ * Constructor.
+ **/
+ public KerberosSecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath) throws IOException {
+ super(secUri, secConfPath);
+ }
+
+ /** {@inheritDoc} */
+ @Override public FileSystem createFileSystem(String userName) throws IOException {
+ UserGroupInformation.setConfiguration(cfg);
+
+ UserGroupInformation ugi = UserGroupInformation.createProxyUser(userName, UserGroupInformation.getCurrentUser());
+
+ try {
+ return ugi.doAs(new PrivilegedExceptionAction<FileSystem>() {
+ @Override public FileSystem run() throws Exception {
+ return FileSystem.get(uri, cfg);
+ }
+ });
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IOException("Failed to create file system due to interrupt.", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
index d5be074..d0326ea 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemsUtils;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.IgniteUtils;
+import static org.apache.ignite.internal.util.typedef.F.*;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -38,10 +39,13 @@ import org.jetbrains.annotations.Nullable;
*/
public class SecondaryFileSystemProvider {
/** Configuration of the secondary filesystem, never null. */
- private final Configuration cfg = HadoopUtils.safeCreateConfiguration();
+ protected final Configuration cfg = HadoopUtils.safeCreateConfiguration();
/** The secondary filesystem URI, never null. */
- private final URI uri;
+ protected final URI uri;
+
+ /** Configuration file path. */
+ @Nullable protected final String confPath;
/**
* Creates new provider with given config parameters. The configuration URL is optional. The filesystem URI must be
@@ -53,15 +57,17 @@ public class SecondaryFileSystemProvider {
* See {@link IgniteUtils#resolveIgniteUrl(String)} on how the path resolved.
* @throws IOException
*/
- public SecondaryFileSystemProvider(final @Nullable String secUri,
- final @Nullable String secConfPath) throws IOException {
- if (secConfPath != null) {
- URL url = U.resolveIgniteUrl(secConfPath);
+ public SecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath) throws IOException {
+ secUri = nullifyEmpty(secUri);
+ confPath = nullifyEmpty(secConfPath);
+
+ if (confPath != null) {
+ URL url = U.resolveIgniteUrl(confPath);
if (url == null) {
// If secConfPath is given, it should be resolvable:
throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
- "(ensure that it exists locally and you have read access to it): " + secConfPath);
+ "(ensure that it exists locally and you have read access to it): " + confPath);
}
cfg.addResource(url);
@@ -90,7 +96,7 @@ public class SecondaryFileSystemProvider {
* @throws IOException
*/
public FileSystem createFileSystem(String userName) throws IOException {
- userName = IgfsUtils.fixUserName(userName);
+ userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
final FileSystem fileSys;
@@ -136,4 +142,11 @@ public class SecondaryFileSystemProvider {
public URI uri() {
return uri;
}
+
+ /**
+ * @return The configuration path, if any.
+ */
+ @Nullable public String configurationPath() {
+ return confPath;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
new file mode 100644
index 0000000..5bbf4d9
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
@@ -0,0 +1,183 @@
+package org.apache.ignite.internal.processors.hadoop.fs;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Collection;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.igfs.HadoopFileSystemFactory;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
+import org.apache.ignite.internal.processors.igfs.IgfsPaths;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lifecycle.LifecycleAware;
+
+import static org.apache.ignite.internal.util.lang.GridFunc.nullifyEmpty;
+
+/**
+ * The class is to be instantiated as a Spring beans, so it must have public zero-arg constructor.
+ * The class is serializable as it will be transferred over the network as a part of {@link IgfsPaths} object.
+ */
+public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<FileSystem>, Externalizable, LifecycleAware {
+ /** Configuration of the secondary filesystem, never null. */
+ protected final Configuration cfg = HadoopUtils.safeCreateConfiguration();
+
+ /** */
+ private URI uri;
+
+ /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
+ private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
+ new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
+ @Override public FileSystem createValue(String key) {
+ try {
+ assert !F.isEmpty(key);
+
+ return createFileSystem(key);
+ }
+ catch (IOException ioe) {
+ throw new IgniteException(ioe);
+ }
+ }
+ }
+ );
+
+ public DefaultHadoopFileSystemFactory() {
+ //
+ }
+
+ @Override public FileSystem get(String userName) throws IOException {
+ return fileSysLazyMap.getOrCreate(userName);
+ }
+
+ public void setUri(URI uri) {
+ this.uri = uri;
+ }
+
+ @Override public URI uri() {
+ return uri;
+ }
+
+// public void setCfg(Configuration cfg) {
+// this.cfg = cfg;
+// }
+
+ /**
+ * Configuration(s) setter, to be invoked from Spring config.
+ * @param cfgPaths
+ */
+ public void setCfgPaths(Collection<String> cfgPaths) {
+ cfgPaths = nullifyEmpty(cfgPaths);
+
+ if (cfgPaths == null)
+ return;
+
+ for (String confPath: cfgPaths) {
+ confPath = nullifyEmpty(confPath);
+
+ if (confPath != null) {
+ URL url = U.resolveIgniteUrl(confPath);
+
+ if (url == null) {
+ // If secConfPath is given, it should be resolvable:
+ throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
+ "(ensure that it exists locally and you have read access to it): " + confPath);
+ }
+
+ cfg.addResource(url);
+ }
+ }
+ }
+
+
+ protected void init() throws IOException {
+ String secUri = nullifyEmpty(uri == null ? null : uri.toString());
+
+ A.ensure(cfg != null, "config");
+
+ // if secondary fs URI is not given explicitly, try to get it from the configuration:
+ if (secUri == null)
+ uri = FileSystem.getDefaultUri(cfg);
+ else {
+ try {
+ uri = new URI(secUri);
+ }
+ catch (URISyntaxException use) {
+ throw new IOException("Failed to resolve secondary file system URI: " + secUri);
+ }
+ }
+
+ // Disable caching:
+ String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(uri.getScheme());
+
+ cfg.setBoolean(prop, true);
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ protected FileSystem createFileSystem(String userName) throws IOException {
+ userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
+
+ final FileSystem fileSys;
+
+ try {
+ fileSys = FileSystem.get(uri, cfg, userName);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IOException("Failed to create file system due to interrupt.", e);
+ }
+
+ return fileSys;
+ }
+
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ cfg.write(out);
+
+ U.writeString(out, uri.toString());
+ }
+
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ cfg.clear();
+
+ cfg.readFields(in);
+
+ String uriStr = U.readString(in);
+
+ try {
+ uri = new URI(uriStr);
+ }
+ catch (URISyntaxException use) {
+ throw new IOException(use);
+ }
+ }
+
+ @Override public void start() throws IgniteException {
+ try {
+ init();
+ }
+ catch (IOException ice) {
+ throw new IgniteException(ice);
+ }
+ }
+
+ @Override public void stop() throws IgniteException {
+ try {
+ fileSysLazyMap.close();
+ }
+ catch (IgniteCheckedException ice) {
+ throw new IgniteException(ice);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
index 608bd25..867351f 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopFileSystemUniversalFileSystemAdapter.java
@@ -42,6 +42,8 @@ public class HadoopFileSystemUniversalFileSystemAdapter implements UniversalFile
* @param fs the filesystem to be wrapped.
*/
public HadoopFileSystemUniversalFileSystemAdapter(FileSystem fs) {
+ assert fs != null;
+
this.fileSys = fs;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
index 0216f4b..eac6bb8 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/testsuites/IgniteHadoopTestSuite.java
@@ -183,7 +183,7 @@ public class IgniteHadoopTestSuite extends TestSuite {
* @throws Exception If failed.
*/
public static void downloadHadoop() throws Exception {
- String ver = IgniteSystemProperties.getString("hadoop.version", "2.4.1");
+ String ver = IgniteSystemProperties.getString("hadoop.version", "2.6.0");
X.println("Will use Hadoop version: " + ver);
http://git-wip-us.apache.org/repos/asf/ignite/blob/7a3f9dd3/parent/pom.xml
----------------------------------------------------------------------
diff --git a/parent/pom.xml b/parent/pom.xml
index f665d40..0481088 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -69,7 +69,7 @@
<guava14.version>14.0.1</guava14.version>
<guava16.version>16.0.1</guava16.version>
<h2.version>1.3.175</h2.version>
- <hadoop.version>2.4.1</hadoop.version>
+ <hadoop.version>2.6.0</hadoop.version>
<httpclient.version>4.5.1</httpclient.version>
<httpcore.version>4.4.3</httpcore.version>
<jackson.version>1.9.13</jackson.version>
[11/12] ignite git commit: IGNITE-2206: WIP.
Posted by vo...@apache.org.
IGNITE-2206: WIP.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/13f8170d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/13f8170d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/13f8170d
Branch: refs/heads/ignite-2206
Commit: 13f8170da81007b4a32eb3918a125df120bb0065
Parents: 2f38e46
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Dec 23 13:17:34 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Dec 23 13:17:34 2015 +0300
----------------------------------------------------------------------
.../igfs/secondary/IgfsSecondaryFileSystem.java | 22 --
.../processors/hadoop/HadoopPayloadAware.java | 22 +-
.../ignite/internal/processors/igfs/IgfsEx.java | 13 --
.../processors/igfs/IgfsHandshakeResponse.java | 3 +-
.../internal/processors/igfs/IgfsImpl.java | 10 +-
.../internal/processors/igfs/IgfsPaths.java | 32 +--
.../igfs/IgfsSecondaryFileSystemImpl.java | 15 --
.../ignite/internal/util/lang/GridFunc.java | 24 ---
.../visor/node/VisorIgfsConfiguration.java | 57 ------
.../fs/CachingHadoopFileSystemFactory.java | 202 +++++++++++++++++++
.../hadoop/fs/HadoopFileSystemFactory.java | 41 ++++
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 45 ++---
.../fs/v1/DefaultHadoopFileSystemFactory.java | 185 -----------------
.../hadoop/fs/v1/HadoopFileSystemFactory.java | 21 --
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 3 +-
.../hadoop/SecondaryFileSystemProvider.java | 5 +-
.../IgniteHadoopFileSystemAbstractSelfTest.java | 8 +-
17 files changed, 307 insertions(+), 401 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index 354b0ae..3f124eb 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -192,26 +192,4 @@ public interface IgfsSecondaryFileSystem {
* @throws IgniteException In case of error.
*/
public long usedSpaceSize() throws IgniteException;
-
-// /**
-// * Gets the implementation specific properties of file system.
-// *
-// * @return Map of properties.
-// * @deprecated Should not be used.
-// */
-// @Deprecated
-// public Map<String,String> properties();
-//
-// /**
-// *
-// * @return The factory.
-// */
-// public @Nullable HadoopFileSystemFactory<F> getSecondaryFileSystemFactory();
-
-
- /**
- * Closes the secondary file system.
- * @throws IgniteException in case of an error.
- */
- public void close() throws IgniteException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
index dcb163f..9b79729 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
@@ -1,12 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.ignite.internal.processors.hadoop;
/**
- *
+ * Gets payload for Hadoop secondary file system.
*/
public interface HadoopPayloadAware {
/**
- *
- * @return
+ * @return Payload.
*/
public Object getPayload();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index a338813..cf268e0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -43,19 +43,6 @@ public interface IgfsEx extends IgniteFileSystem {
/** File property: prefer writes to local node. */
public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
-// /** Property name for path to Hadoop configuration. */
-// public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
-//
-// /** Property name for URI of file system. */
-// public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
-//
-// /** Property name for default user name of file system.
-// * NOTE: for secondary file system this is just a default user name, which is used
-// * when the 2ndary filesystem is used outside of any user context.
-// * If another user name is set in the context, 2ndary file system will work on behalf
-// * of that user, which is different from the default. */
-// public static final String SECONDARY_FS_USER_NAME = "SECONDARY_FS_USER_NAME";
-
/**
* Stops IGFS cleaning all used resources.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
index 52cae8f..1ba98ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
@@ -55,8 +55,7 @@ public class IgfsHandshakeResponse implements Externalizable {
* @param paths Secondary paths.
* @param blockSize Server default block size.
*/
- public IgfsHandshakeResponse(String igfsName, IgfsPaths paths,
- long blockSize, Boolean sampling) {
+ public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, long blockSize, Boolean sampling) {
assert paths != null;
this.igfsName = igfsName;
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 7453e15..a44329a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -264,12 +264,7 @@ public final class IgfsImpl implements IgfsEx {
if (secondaryFs instanceof HadoopPayloadAware)
secondaryFsPayload = ((HadoopPayloadAware) secondaryFs).getPayload();
- secondaryPaths = new IgfsPaths(
- //secondaryFs == null ? null : secondaryFs.properties(),
- //secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
- secondaryFsPayload,
- dfltMode,
- modeRslvr.modesOrdered());
+ secondaryPaths = new IgfsPaths(secondaryFsPayload, dfltMode, modeRslvr.modesOrdered());
// Check whether IGFS LRU eviction policy is set on data cache.
String dataCacheName = igfsCtx.configuration().getDataCacheName();
@@ -317,7 +312,8 @@ public final class IgfsImpl implements IgfsEx {
batch.cancel();
try {
- secondaryFs.close();
+ if (secondaryFs instanceof LifecycleAware)
+ ((LifecycleAware)secondaryFs).stop();
}
catch (Exception e) {
log.error("Failed to close secondary file system.", e);
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index 83451db..986f59f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -59,12 +59,11 @@ public class IgfsPaths implements Externalizable {
/**
* Constructor.
*
+ * @param payload Payload.
* @param dfltMode Default IGFS mode.
* @param pathModes Path modes.
*/
- public IgfsPaths(Object payload,
- IgfsMode dfltMode,
- @Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
+ public IgfsPaths(Object payload, IgfsMode dfltMode, @Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
this.payload = payload;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
@@ -84,10 +83,15 @@ public class IgfsPaths implements Externalizable {
return pathModes;
}
+ /**
+ * @return Payload.
+ */
+ @Nullable public Object getPayload() {
+ return payload;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
-// U.writeStringMap(out, props);
-
writePayload(out);
U.writeEnum(out, dfltMode);
@@ -106,9 +110,10 @@ public class IgfsPaths implements Externalizable {
}
/**
+ * Write payload.
*
- * @param out
- * @throws IOException
+ * @param out Output stream.
+ * @throws IOException If failed.
*/
private void writePayload(ObjectOutput out) throws IOException {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -127,8 +132,6 @@ public class IgfsPaths implements Externalizable {
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
-// props = U.readStringMap(in);
-
readPayload(in);
dfltMode = IgfsMode.fromOrdinal(in.readByte());
@@ -150,10 +153,11 @@ public class IgfsPaths implements Externalizable {
}
/**
+ * Read payload.
*
- * @param in
- * @throws IOException
- * @throws ClassNotFoundException
+ * @param in Input stream.
+ * @throws IOException If failed.
+ * @throws ClassNotFoundException If failed.
*/
private void readPayload(ObjectInput in) throws IOException, ClassNotFoundException {
byte[] factoryBytes = U.readByteArray(in);
@@ -167,8 +171,4 @@ public class IgfsPaths implements Externalizable {
oi.close();
}
}
-
- public Object getPayload() {
- return payload;
- }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
index ced6b21..44e858f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
@@ -115,19 +115,4 @@ class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
@Override public long usedSpaceSize() throws IgniteException {
return igfs.usedSpaceSize();
}
-
-// /** {@inheritDoc} */
-// @Override public Map<String, String> properties() {
-// return Collections.emptyMap();
-// }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteException {
- // No-op.
- }
-
-// /** {@inheritDoc} */
-// @Override public HadoopFileSystemFactory<IgfsEx> getSecondaryFileSystemFactory() {
-// return null;
-// }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
index 224d205..8d5a8e7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/util/lang/GridFunc.java
@@ -2290,30 +2290,6 @@ public class GridFunc {
}
/**
- * Nullifies an empty String.
- * @param x The argument.
- * @return Nullified argument.
- */
- public static String nullifyEmpty(String x) {
- if (isEmpty(x))
- return null;
-
- return x;
- }
-
- /**
- * Nullifies an empty collection.
- * @param c The argument.
- * @return Nullified argument.
- */
- public static <T> Collection<T> nullifyEmpty(Collection<T> c) {
- if (isEmpty(c))
- return null;
-
- return c;
- }
-
- /**
* Utility map getter. This method analogous to {@link #addIfAbsent(Map, Object, Callable)}
* method but this one doesn't put the default value into the map when key is not found.
*
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 2238fdf..ea0e721 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -29,9 +29,6 @@ import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.jetbrains.annotations.Nullable;
-//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
-//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_USER_NAME;
import static org.apache.ignite.internal.visor.util.VisorTaskUtils.compactClass;
/**
@@ -65,17 +62,6 @@ public class VisorIgfsConfiguration implements Serializable {
/** Number of batches that can be concurrently sent to remote node. */
private int perNodeParallelBatchCnt;
-// /** URI of the secondary Hadoop file system. */
-// private String secondaryHadoopFileSysUri;
-//
-// /** Path for the secondary hadoop file system config. */
-// private String secondaryHadoopFileSysCfgPath;
-//
-// /** User name for the secondary hadoop file system config. */
-// private String secondaryHadoopFileSysUserName;
-
-// private HadoopFileSystemFactory factory;
-
/** IGFS instance mode. */
private IgfsMode dfltMode;
@@ -143,19 +129,6 @@ public class VisorIgfsConfiguration implements Serializable {
cfg.perNodeBatchSize = igfs.getPerNodeBatchSize();
cfg.perNodeParallelBatchCnt = igfs.getPerNodeParallelBatchCount();
- IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
-
-// if (secFs != null) {
-// //Map<String, String> props = secFs.properties();
-//
-// //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
-// //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
-// //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
-//
-// // Just take and save the factory object:
-// cfg.factory = secFs.getSecondaryFileSystemFactory();
-// }
-
cfg.dfltMode = igfs.getDefaultMode();
cfg.pathModes = igfs.getPathModes();
cfg.dualModePutExecutorSrvc = compactClass(igfs.getDualModePutExecutorService());
@@ -255,36 +228,6 @@ public class VisorIgfsConfiguration implements Serializable {
return perNodeParallelBatchCnt;
}
-// /**
-// * @return URI of the secondary Hadoop file system.
-// */
-// @Nullable public String secondaryHadoopFileSystemUri() {
-// return secondaryHadoopFileSysUri;
-// }
-//
-// /**
-// * @return User name of the secondary Hadoop file system.
-// */
-// @Nullable public String secondaryHadoopFileSystemUserName() {
-// return secondaryHadoopFileSysUserName;
-// }
-//
-// /**
-// * @return Path for the secondary hadoop file system config.
-// */
-// @Nullable public String secondaryHadoopFileSystemConfigPath() {
-// return secondaryHadoopFileSysCfgPath;
-// }
-
-// /**
-// *
-// * @param <T>
-// * @return
-// */
-// public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
-// return factory;
-// }
-
/**
* @return IGFS instance mode.
*/
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
new file mode 100644
index 0000000..3bad850
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
+import org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemsUtils;
+import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
+import org.apache.ignite.internal.processors.igfs.IgfsPaths;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lifecycle.LifecycleAware;
+
+/**
+ * The class is to be instantiated as a Spring beans, so it must have public zero-arg constructor.
+ * The class is serializable as it will be transferred over the network as a part of {@link IgfsPaths} object.
+ */
+public class CachingHadoopFileSystemFactory implements HadoopFileSystemFactory, Externalizable, LifecycleAware {
+ /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
+ private final transient HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
+ new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
+ @Override public FileSystem createValue(String key) {
+ try {
+ assert !F.isEmpty(key);
+
+ return createFileSystem(key);
+ }
+ catch (IOException ioe) {
+ throw new IgniteException(ioe);
+ }
+ }
+ }
+ );
+
+ /** Configuration of the secondary filesystem, never null. */
+ protected transient Configuration cfg;
+
+ /** */
+ protected transient URI uri;
+
+ /** */
+ protected String uriStr;
+
+ /** */
+ protected List<String> cfgPathStr;
+
+ int getCount = 0;
+
+ /**
+ *
+ */
+ public CachingHadoopFileSystemFactory() {
+ //
+
+
+
+ }
+
+ @Override public FileSystem create(String userName) throws IOException {
+ A.ensure(cfg != null, "cfg");
+
+ if (getCount == 0)
+ assert fileSysLazyMap.size() == 0;
+
+ getCount++;
+
+ return fileSysLazyMap.getOrCreate(userName);
+ }
+
+ // TODO: Add getter.
+
+ /**
+ * Uri setter.
+ * @param uriStr
+ */
+ public void setUri(String uriStr) {
+ this.uriStr = uriStr;
+ }
+
+ // TODO: Add getter.
+
+ /**
+ * Configuration(s) setter, to be invoked from Spring config.
+ * @param cfgPaths
+ */
+ public void setConfigPaths(List<String> cfgPaths) {
+ this.cfgPathStr = cfgPaths;
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ protected FileSystem createFileSystem(String userName) throws IOException {
+ userName = IgfsUtils.fixUserName(userName);
+
+ assert cfg != null;
+
+ final FileSystem fileSys;
+
+ try {
+ fileSys = FileSystem.get(uri, cfg, userName);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IOException("Failed to create file system due to interrupt.", e);
+ }
+
+ return fileSys;
+ }
+
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeString(out, uriStr);
+
+ U.writeCollection(out, cfgPathStr);
+ }
+
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ uriStr = U.readString(in);
+
+ cfgPathStr = new ArrayList(U.readCollection(in));
+ }
+
+ @Override public void start() throws IgniteException {
+ cfg = HadoopUtils.safeCreateConfiguration();
+
+ if (cfgPathStr != null) {
+ for (String confPath : cfgPathStr) {
+ if (confPath != null) {
+ URL url = U.resolveIgniteUrl(confPath);
+
+ if (url == null) {
+ // If secConfPath is given, it should be resolvable:
+ throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
+
+ "(ensure that it exists locally and you have read access to it): " + confPath);
+ }
+
+ cfg.addResource(url);
+ }
+ }
+ }
+
+ // if secondary fs URI is not given explicitly, try to get it from the configuration:
+ if (uriStr == null)
+ uri = FileSystem.getDefaultUri(cfg);
+ else {
+ try {
+ uri = new URI(uriStr);
+ }
+ catch (URISyntaxException use) {
+ throw new IgniteException("Failed to resolve secondary file system URI: " + uriStr);
+ }
+ }
+
+ assert uriStr != null;
+
+ // Disable caching:
+ String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(uri.getScheme());
+
+ cfg.setBoolean(prop, true);
+ }
+
+ @Override public void stop() throws IgniteException {
+ try {
+ fileSysLazyMap.close();
+ }
+ catch (IgniteCheckedException ice) {
+ throw new IgniteException(ice);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
new file mode 100644
index 0000000..b4edab8
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * This factory is {@link Serializable} because it should be transferable over the network.
+ * <p>
+ * Implementations may choose not to construct a new instance, but instead
+ * return a previously created instance.
+ */
+public interface HadoopFileSystemFactory extends Serializable {
+ /**
+ * Creates the file system, possibly taking a cached instance.
+ * All the other data needed for the file system creation are expected to be contained
+ * in this object instance.
+ *
+ * @param userName The user name
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public FileSystem create(String userName) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 86ed7a0..756989c 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.hadoop.fs.v1.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -47,14 +46,12 @@ import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
import org.apache.ignite.internal.processors.hadoop.HadoopPayloadAware;
-import org.apache.ignite.hadoop.fs.v1.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable;
import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
import org.apache.ignite.internal.processors.igfs.IgfsFileInfo;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.F;
-import static org.apache.ignite.internal.util.typedef.F.*;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.lifecycle.LifecycleAware;
@@ -95,7 +92,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
// private @Nullable Collection<URI> cfgPaths;
/** The default user name. It is used if no user context is set. */
- private String dfltUserName = IgfsUtils.fixUserName(null);
+ private String usrName = IgfsUtils.fixUserName(null);
/** */
private HadoopFileSystemFactory fsFactory;
@@ -160,33 +157,22 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
@Deprecated
public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath,
@Nullable String userName) throws IgniteCheckedException {
+ CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();
- uri = nullifyEmpty(uri);
-// if (uri != null)
-// U.warn(null, "This constructor is deprecated. URI value passed in will be ignored.");
-
- cfgPath = nullifyEmpty(cfgPath);
-// if (cfgPath != null)
-// U.warn(null, "This constructor is deprecated. The configurationPath value passed in will be ignored.");
-
- DefaultHadoopFileSystemFactory fac = new DefaultHadoopFileSystemFactory();
-
- if (uri != null)
- fac.setUri(uri);
+ fac.setUri(uri);
if (cfgPath != null)
fac.setConfigPaths(Collections.singletonList(cfgPath));
- setFsFactory(fac);
-
- setDfltUserName(userName);
+ setFileSystemFactory(fac);
+ setUserName(userName);
}
/**
*
* @param factory
*/
- public void setFsFactory(HadoopFileSystemFactory factory) {
+ public void setFileSystemFactory(HadoopFileSystemFactory factory) {
A.ensure(factory != null, "Factory value must not be null.");
this.fsFactory = factory;
@@ -194,10 +180,11 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/**
*
- * @param dfltUserName
+ * @param usrName
*/
- public void setDfltUserName(String dfltUserName) {
- this.dfltUserName = IgfsUtils.fixUserName(nullifyEmpty(dfltUserName));
+ public void setUserName(String usrName) {
+ // TODO: Move fix to start routine.
+ this.usrName = IgfsUtils.fixUserName(usrName);
}
// /**
@@ -580,17 +567,17 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
String user = IgfsUserContext.currentUser();
if (F.isEmpty(user))
- user = dfltUserName; // default is never empty.
+ user = usrName; // default is never empty.
assert !F.isEmpty(user);
- if (F.eq(user, dfltUserName))
+ if (F.eq(user, usrName))
return dfltFs; // optimization
//assert fsFactory.uri() != null : "uri!";
try {
- return fsFactory.get(user);
+ return fsFactory.create(user);
}
catch (IOException ioe) {
throw new IgniteException(ioe);
@@ -605,7 +592,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
@Override public void start() {
// #start() should not ever be invoked if these properties are not set:
A.ensure(fsFactory != null, "factory");
- A.ensure(dfltUserName != null, "dfltUserName");
+ A.ensure(usrName != null, "dfltUserName");
// Avoid
if (started.compareAndSet(false, true)) {
@@ -618,7 +605,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
// File system creation for the default user name.
// The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
//this.dfltFs = secProvider.createFileSystem(dfltUserName);
- this.dfltFs = fsFactory.get(dfltUserName);
+ this.dfltFs = fsFactory.create(usrName);
assert dfltFs != null;
}
@@ -633,6 +620,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
// return fsFactory;
// }
+
+
@Override public void stop() throws IgniteException {
close();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
deleted file mode 100644
index ba4bfdd..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
+++ /dev/null
@@ -1,185 +0,0 @@
-package org.apache.ignite.hadoop.fs.v1;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
-import org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemsUtils;
-import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
-import org.apache.ignite.internal.processors.igfs.IgfsPaths;
-import org.apache.ignite.internal.processors.igfs.IgfsUtils;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lifecycle.LifecycleAware;
-
-import static org.apache.ignite.internal.util.lang.GridFunc.nullifyEmpty;
-
-/**
- * The class is to be instantiated as a Spring beans, so it must have public zero-arg constructor.
- * The class is serializable as it will be transferred over the network as a part of {@link IgfsPaths} object.
- */
-public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory, Externalizable, LifecycleAware {
- /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
- private final transient HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
- new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
- @Override public FileSystem createValue(String key) {
- try {
- assert !F.isEmpty(key);
-
- return createFileSystem(key);
- }
- catch (IOException ioe) {
- throw new IgniteException(ioe);
- }
- }
- }
- );
-
- /** Configuration of the secondary filesystem, never null. */
- protected transient Configuration cfg;
-
- /** */
- protected transient URI uri;
-
- /** */
- protected String uriStr;
-
- /** */
- protected List<String> cfgPathStr;
-
- int getCount = 0;
-
- /**
- *
- */
- public DefaultHadoopFileSystemFactory() {
- //
-
-
-
- }
-
- @Override public FileSystem get(String userName) throws IOException {
- A.ensure(cfg != null, "cfg");
-
- if (getCount == 0)
- assert fileSysLazyMap.size() == 0;
-
- getCount++;
-
- return fileSysLazyMap.getOrCreate(userName);
- }
-
- /**
- * Uri setter.
- * @param uriStr
- */
- public void setUri(String uriStr) {
- this.uriStr = uriStr;
- }
-
- /**
- * Configuration(s) setter, to be invoked from Spring config.
- * @param cfgPaths
- */
- public void setConfigPaths(List<String> cfgPaths) {
- this.cfgPathStr = (List)nullifyEmpty(cfgPaths);
- }
-
- /**
- * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
- * @throws IOException
- */
- protected FileSystem createFileSystem(String userName) throws IOException {
- userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
-
- assert cfg != null;
-
- final FileSystem fileSys;
-
- try {
- fileSys = FileSystem.get(uri, cfg, userName);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IOException("Failed to create file system due to interrupt.", e);
- }
-
- return fileSys;
- }
-
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeString(out, uriStr);
-
- U.writeCollection(out, cfgPathStr);
- }
-
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- uriStr = U.readString(in);
-
- cfgPathStr = new ArrayList(U.readCollection(in));
- }
-
- @Override public void start() throws IgniteException {
- cfg = HadoopUtils.safeCreateConfiguration();
-
- if (cfgPathStr != null) {
- for (String confPath : cfgPathStr) {
- confPath = nullifyEmpty(confPath);
-
- if (confPath != null) {
- URL url = U.resolveIgniteUrl(confPath);
-
- if (url == null) {
- // If secConfPath is given, it should be resolvable:
- throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
-
- "(ensure that it exists locally and you have read access to it): " + confPath);
- }
-
- cfg.addResource(url);
- }
- }
- }
-
- // if secondary fs URI is not given explicitly, try to get it from the configuration:
- if (uriStr == null)
- uri = FileSystem.getDefaultUri(cfg);
- else {
- try {
- uri = new URI(uriStr);
- }
- catch (URISyntaxException use) {
- throw new IgniteException("Failed to resolve secondary file system URI: " + uriStr);
- }
- }
-
- assert uriStr != null;
-
- // Disable caching:
- String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(uri.getScheme());
-
- cfg.setBoolean(prop, true);
- }
-
- @Override public void stop() throws IgniteException {
- try {
- fileSysLazyMap.close();
- }
- catch (IgniteCheckedException ice) {
- throw new IgniteException(ice);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
deleted file mode 100644
index c1c7b9d..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package org.apache.ignite.hadoop.fs.v1;
-
-import java.io.IOException;
-import java.io.Serializable;
-import org.apache.hadoop.fs.FileSystem;
-
-/**
- * This factory is {@link Serializable} because it should be transferable over the network.
- */
-public interface HadoopFileSystemFactory extends Serializable {
- /**
- * Gets the file system, possibly creating it or taking a cached instance.
- * All the other data needed for the file system creation are expected to be contained
- * in this object instance.
- *
- * @param userName The user name
- * @return The file system.
- * @throws IOException On error.
- */
- public FileSystem get(String userName) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 355892e..9463412 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -344,7 +345,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
((LifecycleAware) factory).start();
try {
- secondaryFs = factory.get(user);
+ secondaryFs = factory.create(user);
secondaryUri = secondaryFs.getUri();
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
index d0326ea..1e7ac7f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
@@ -58,8 +58,7 @@ public class SecondaryFileSystemProvider {
* @throws IOException
*/
public SecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath) throws IOException {
- secUri = nullifyEmpty(secUri);
- confPath = nullifyEmpty(secConfPath);
+ confPath = secConfPath;
if (confPath != null) {
URL url = U.resolveIgniteUrl(confPath);
@@ -96,7 +95,7 @@ public class SecondaryFileSystemProvider {
* @throws IOException
*/
public FileSystem createFileSystem(String userName) throws IOException {
- userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
+ userName = IgfsUtils.fixUserName(userName);
final FileSystem fileSys;
http://git-wip-us.apache.org/repos/asf/ignite/blob/13f8170d/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 1ce0492..310c390 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -62,7 +62,7 @@ import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
-import org.apache.ignite.hadoop.fs.v1.DefaultHadoopFileSystemFactory;
+import org.apache.ignite.hadoop.fs.CachingHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEx;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsIpcIo;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutProc;
@@ -383,15 +383,15 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
cfg.setDefaultMode(mode);
if (mode != PRIMARY) {
- DefaultHadoopFileSystemFactory fac = new DefaultHadoopFileSystemFactory();
+ CachingHadoopFileSystemFactory fac = new CachingHadoopFileSystemFactory();
fac.setUri(SECONDARY_URI);
fac.setConfigPaths(Collections.singletonList(SECONDARY_CFG_PATH));
IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem();
- sec.setFsFactory(fac);
- sec.setDfltUserName(SECONDARY_FS_USER);
+ sec.setFileSystemFactory(fac);
+ sec.setUserName(SECONDARY_FS_USER);
// NB: start() will be invoked upon IgfsImpl init.
cfg.setSecondaryFileSystem(sec);
[07/12] ignite git commit: Merge branch 'ignite-1.5' of
https://github.com/apache/ignite into ignite-2206
Posted by vo...@apache.org.
Merge branch 'ignite-1.5' of https://github.com/apache/ignite into ignite-2206
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ba11b3a7
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ba11b3a7
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ba11b3a7
Branch: refs/heads/ignite-2206
Commit: ba11b3a76156d5a40b47b6102d0219fb01c1922d
Parents: d12b9f1 8be0ced
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 18:18:36 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 18:18:36 2015 +0300
----------------------------------------------------------------------
assembly/dependencies-fabric-lgpl.xml | 7 +++
assembly/dependencies-fabric.xml | 7 +++
examples/pom.xml | 2 +-
examples/schema-import/pom.xml | 2 +-
modules/aop/pom.xml | 2 +-
modules/apache-license-gen/pom.xml | 2 +-
modules/aws/pom.xml | 2 +-
modules/camel/pom.xml | 2 +-
modules/clients/pom.xml | 2 +-
modules/cloud/pom.xml | 2 +-
modules/codegen/pom.xml | 2 +-
modules/core/pom.xml | 2 +-
.../ignite/internal/IgniteVersionUtils.java | 5 +-
.../ignite/lang/IgniteProductVersion.java | 2 +-
.../core/src/main/resources/ignite.properties | 2 +-
.../product/GridProductVersionSelfTest.java | 22 ++++++---
modules/extdata/p2p/pom.xml | 2 +-
.../extdata/uri/modules/uri-dependency/pom.xml | 2 +-
modules/extdata/uri/pom.xml | 2 +-
modules/flume/pom.xml | 2 +-
modules/gce/pom.xml | 2 +-
modules/geospatial/pom.xml | 2 +-
modules/hadoop/pom.xml | 2 +-
modules/hibernate/pom.xml | 2 +-
modules/indexing/pom.xml | 2 +-
modules/jcl/pom.xml | 2 +-
modules/jms11/pom.xml | 2 +-
modules/jta/pom.xml | 2 +-
modules/kafka/pom.xml | 2 +-
modules/log4j/pom.xml | 2 +-
modules/log4j2/pom.xml | 2 +-
modules/mesos/pom.xml | 2 +-
modules/mqtt/pom.xml | 2 +-
modules/osgi-karaf/pom.xml | 2 +-
modules/osgi-paxlogging/pom.xml | 2 +-
modules/osgi/pom.xml | 2 +-
modules/platforms/cpp/common/configure.ac | 2 +-
modules/platforms/cpp/core-test/configure.ac | 2 +-
modules/platforms/cpp/core/configure.ac | 2 +-
modules/platforms/cpp/examples/configure.ac | 2 +-
modules/platforms/cpp/ignite/configure.ac | 2 +-
.../Properties/AssemblyInfo.cs | 6 +--
.../Properties/AssemblyInfo.cs | 6 +--
.../Properties/AssemblyInfo.cs | 6 +--
.../Properties/AssemblyInfo.cs | 6 +--
.../Apache.Ignite/Properties/AssemblyInfo.cs | 6 +--
.../Properties/AssemblyInfo.cs | 6 +--
.../Properties/AssemblyInfo.cs | 6 +--
modules/rest-http/pom.xml | 2 +-
modules/scalar-2.10/pom.xml | 2 +-
modules/scalar/pom.xml | 2 +-
modules/schedule/pom.xml | 2 +-
modules/schema-import/pom.xml | 2 +-
modules/slf4j/pom.xml | 2 +-
modules/spark-2.10/pom.xml | 2 +-
modules/spark/pom.xml | 2 +-
modules/spring/pom.xml | 2 +-
modules/ssh/pom.xml | 2 +-
modules/tools/pom.xml | 2 +-
modules/twitter/pom.xml | 2 +-
modules/urideploy/pom.xml | 2 +-
modules/visor-console-2.10/pom.xml | 2 +-
modules/visor-console/pom.xml | 2 +-
modules/visor-plugins/pom.xml | 2 +-
modules/web/pom.xml | 2 +-
.../config/benchmark-multicast.properties | 6 +--
modules/yardstick/pom.xml | 2 +-
.../cache/IgnitePutTxImplicitBenchmark.java | 52 ++++++++++++++++++++
.../cache/IgnitePutTxOffHeapBenchmark.java | 2 +-
.../IgnitePutTxOffHeapValuesBenchmark.java | 2 +-
modules/yarn/pom.xml | 2 +-
modules/zookeeper/pom.xml | 2 +-
pom.xml | 2 +-
73 files changed, 170 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
[09/12] ignite git commit: Merge branch 'ignite-1.5' of
https://github.com/apache/ignite into ignite-2206
Posted by vo...@apache.org.
Merge branch 'ignite-1.5' of https://github.com/apache/ignite into ignite-2206
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/e5b92acb
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/e5b92acb
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/e5b92acb
Branch: refs/heads/ignite-2206
Commit: e5b92acb461926ad8511234bcb5c4a3e6de30cb1
Parents: 8969002 b6dab09
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 22:51:36 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 22:51:36 2015 +0300
----------------------------------------------------------------------
modules/platforms/cpp/common/configure.ac | 2 +-
modules/platforms/cpp/core-test/configure.ac | 2 +-
modules/platforms/cpp/core/configure.ac | 2 +-
modules/platforms/cpp/examples/configure.ac | 2 +-
modules/platforms/cpp/ignite/configure.ac | 2 +-
.../dotnet/Apache.Ignite.Benchmarks/Properties/AssemblyInfo.cs | 4 ++--
.../Apache.Ignite.Core.Tests.TestDll/Properties/AssemblyInfo.cs | 4 ++--
.../dotnet/Apache.Ignite.Core.Tests/Properties/AssemblyInfo.cs | 4 ++--
.../dotnet/Apache.Ignite.Core/Properties/AssemblyInfo.cs | 4 ++--
.../platforms/dotnet/Apache.Ignite/Properties/AssemblyInfo.cs | 4 ++--
.../examples/Apache.Ignite.Examples/Properties/AssemblyInfo.cs | 4 ++--
.../Apache.Ignite.ExamplesDll/Properties/AssemblyInfo.cs | 4 ++--
12 files changed, 19 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
[10/12] ignite git commit: Merge branch 'ignite-2206' of
https://github.com/iveselovskiy/ignite into ignite-2206
Posted by vo...@apache.org.
Merge branch 'ignite-2206' of https://github.com/iveselovskiy/ignite into ignite-2206
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2f38e463
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2f38e463
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2f38e463
Branch: refs/heads/ignite-2206
Commit: 2f38e4636c85ee8656daa86d5a3171acdae36235
Parents: 66b33bc e5b92ac
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Dec 23 12:53:28 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Dec 23 12:53:28 2015 +0300
----------------------------------------------------------------------
.../igfs/secondary/IgfsSecondaryFileSystem.java | 20 +-
.../processors/hadoop/HadoopPayloadAware.java | 12 +
.../ignite/internal/processors/igfs/IgfsEx.java | 24 +-
.../processors/igfs/IgfsHandshakeResponse.java | 3 +-
.../internal/processors/igfs/IgfsImpl.java | 16 +-
.../internal/processors/igfs/IgfsPaths.java | 77 ++++--
.../igfs/IgfsSecondaryFileSystemImpl.java | 14 +-
.../ignite/internal/util/lang/GridFunc.java | 24 ++
.../visor/node/VisorIgfsConfiguration.java | 88 ++++---
.../org/apache/ignite/hadoop/HadoopFsIssue.java | 71 ++++++
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 238 +++++++++++++------
.../fs/v1/DefaultHadoopFileSystemFactory.java | 185 ++++++++++++++
.../hadoop/fs/v1/HadoopFileSystemFactory.java | 21 ++
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 59 +++--
.../fs/v2/HadoopAbstractFileSystemFactory.java | 21 ++
.../hadoop/fs/v2/HadoopV2FileSystemFactory.java | 11 +
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 32 ++-
.../hadoop/IgfsSecondaryFileSystemEx.java | 15 ++
.../KerberosSecondaryFileSystemProvider.java | 55 +++++
.../hadoop/SecondaryFileSystemProvider.java | 29 ++-
.../hadoop/fs/HadoopLazyConcurrentMap.java | 4 +
...oopFileSystemUniversalFileSystemAdapter.java | 2 +
.../IgniteHadoopFileSystemAbstractSelfTest.java | 22 +-
.../testsuites/IgniteHadoopTestSuite.java | 2 +-
parent/pom.xml | 2 +-
25 files changed, 862 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
[06/12] ignite git commit: IGNITE-2206: intermadiate for review.
Posted by vo...@apache.org.
IGNITE-2206: intermadiate for review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d12b9f12
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d12b9f12
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d12b9f12
Branch: refs/heads/ignite-2206
Commit: d12b9f1288103dc387080d84256d8b9b14ef5a6b
Parents: 4553562
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 18:18:13 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 18:18:13 2015 +0300
----------------------------------------------------------------------
.../igfs/secondary/IgfsSecondaryFileSystem.java | 31 +++--
.../processors/hadoop/PayloadAware.java | 4 +
.../processors/igfs/IgfsHandshakeResponse.java | 18 +--
.../internal/processors/igfs/IgfsImpl.java | 17 ++-
.../internal/processors/igfs/IgfsPaths.java | 128 ++++++++++---------
.../igfs/IgfsSecondaryFileSystemImpl.java | 20 ++-
.../visor/node/VisorIgfsConfiguration.java | 1 -
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 33 +++--
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 53 ++++----
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 17 +--
.../fs/DefaultHadoopFileSystemFactory.java | 6 +-
11 files changed, 163 insertions(+), 165 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index 696c81a..354b0ae 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -21,7 +21,6 @@ import java.io.OutputStream;
import java.util.Collection;
import java.util.Map;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.jetbrains.annotations.Nullable;
@@ -29,7 +28,7 @@ import org.jetbrains.annotations.Nullable;
/**
* Secondary file system interface.
*/
-public interface IgfsSecondaryFileSystem <F> {
+public interface IgfsSecondaryFileSystem {
/**
* Checks if the specified path exists.
*
@@ -194,20 +193,20 @@ public interface IgfsSecondaryFileSystem <F> {
*/
public long usedSpaceSize() throws IgniteException;
- /**
- * Gets the implementation specific properties of file system.
- *
- * @return Map of properties.
- * @deprecated Should not be used.
- */
- @Deprecated
- public Map<String,String> properties();
-
- /**
- *
- * @return The factory.
- */
- public @Nullable HadoopFileSystemFactory<F> getSecondaryFileSystemFactory();
+// /**
+// * Gets the implementation specific properties of file system.
+// *
+// * @return Map of properties.
+// * @deprecated Should not be used.
+// */
+// @Deprecated
+// public Map<String,String> properties();
+//
+// /**
+// *
+// * @return The factory.
+// */
+// public @Nullable HadoopFileSystemFactory<F> getSecondaryFileSystemFactory();
/**
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
index 24a8545..dc4ff5e 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
@@ -5,5 +5,9 @@ package org.apache.ignite.internal.processors.hadoop;
*/
public interface PayloadAware <P> {
+ /**
+ *
+ * @return
+ */
public P getPayload();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
index 230f798..52cae8f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
@@ -36,9 +36,6 @@ public class IgfsHandshakeResponse implements Externalizable {
/** SECONDARY paths. */
private IgfsPaths paths;
- /** */
- private byte[] secondaryFileSystemFactoryBytes;
-
/** Server block size. */
private long blockSize;
@@ -58,12 +55,11 @@ public class IgfsHandshakeResponse implements Externalizable {
* @param paths Secondary paths.
* @param blockSize Server default block size.
*/
- public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, byte[] secondaryFileSystemFactoryBytes,
+ public IgfsHandshakeResponse(String igfsName, IgfsPaths paths,
long blockSize, Boolean sampling) {
assert paths != null;
this.igfsName = igfsName;
- this.secondaryFileSystemFactoryBytes = secondaryFileSystemFactoryBytes;
this.paths = paths;
this.blockSize = blockSize;
this.sampling = sampling;
@@ -97,22 +93,12 @@ public class IgfsHandshakeResponse implements Externalizable {
return sampling;
}
- /**
- *
- * @return
- */
- public byte[] getSecondaryFileSystemFactoryBytes() {
- return secondaryFileSystemFactoryBytes;
- }
-
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeString(out, igfsName);
paths.writeExternal(out);
- U.writeByteArray(out, secondaryFileSystemFactoryBytes);
-
out.writeLong(blockSize);
if (sampling != null) {
@@ -131,8 +117,6 @@ public class IgfsHandshakeResponse implements Externalizable {
paths.readExternal(in);
- secondaryFileSystemFactoryBytes = U.readByteArray(in);
-
blockSize = in.readLong();
if (in.readBoolean())
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 15509ab..fb93ea1 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.igfs;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
@@ -72,6 +73,7 @@ import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
+import org.apache.ignite.internal.processors.hadoop.PayloadAware;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -87,6 +89,7 @@ import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteBiTuple;
import org.apache.ignite.lang.IgniteFuture;
import org.apache.ignite.lang.IgniteUuid;
+import org.apache.ignite.lifecycle.LifecycleAware;
import org.apache.ignite.resources.IgniteInstanceResource;
import org.apache.ignite.thread.IgniteThreadPoolExecutor;
import org.jetbrains.annotations.Nullable;
@@ -122,7 +125,7 @@ public final class IgfsImpl implements IgfsEx {
static final Map<String, String> DFLT_DIR_META = F.asMap(PROP_PERMISSION, PERMISSION_DFLT_VAL);
/** Handshake message. */
- private final IgfsPaths secondaryPaths;
+ private final IgfsPaths<Serializable> secondaryPaths;
/** Cache based structure (meta data) manager. */
private IgfsMetaManager meta;
@@ -200,6 +203,9 @@ public final class IgfsImpl implements IgfsEx {
data = igfsCtx.data();
secondaryFs = cfg.getSecondaryFileSystem();
+ if (secondaryFs instanceof LifecycleAware)
+ ((LifecycleAware) secondaryFs).start();
+
/* Default IGFS mode. */
IgfsMode dfltMode;
@@ -254,9 +260,16 @@ public final class IgfsImpl implements IgfsEx {
modeRslvr = new IgfsModeResolver(dfltMode, modes);
+ Serializable secondaryFsPayload = null;
+
+ if (secondaryFs instanceof PayloadAware) {
+ secondaryFsPayload = ((PayloadAware<Serializable>) secondaryFs).getPayload();
+ }
+
secondaryPaths = new IgfsPaths(
- secondaryFs == null ? null : secondaryFs.properties(),
+ //secondaryFs == null ? null : secondaryFs.properties(),
//secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
+ secondaryFsPayload,
dfltMode,
modeRslvr.modesOrdered());
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index a9813d0..d434d01 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -25,12 +25,12 @@ import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
-//import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
+import org.apache.ignite.internal.processors.hadoop.PayloadAware;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -38,16 +38,16 @@ import org.jetbrains.annotations.Nullable;
/**
* Description of path modes.
*/
-public class IgfsPaths <F> implements Externalizable {
+public class IgfsPaths <P extends Serializable> implements Externalizable, PayloadAware<P> {
/** */
private static final long serialVersionUID = 0L;
- /** Additional secondary file system properties. */
- @Deprecated
- private Map<String, String> props;
+// /** Additional secondary file system properties. */
+// @Deprecated
+// private Map<String, String> props;
-// /** */
-// private HadoopFileSystemFactory<F> factory;
+ /** */
+ private P payload;
/** Default IGFS mode. */
private IgfsMode dfltMode;
@@ -65,29 +65,28 @@ public class IgfsPaths <F> implements Externalizable {
/**
* Constructor.
*
- * @param props Additional secondary file system properties.
* @param dfltMode Default IGFS mode.
* @param pathModes Path modes.
*/
- public IgfsPaths(Map<String, String> props,
- //HadoopFileSystemFactory<F> factory,
+ public IgfsPaths(//Map<String, String> props,
+ P payload,
IgfsMode dfltMode,
@Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
- this.props = props;
- //this.factory = factory;
+ //this.props = props;
+ this.payload = payload;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
}
- /**
- * @return Secondary file system properties.
- *
- * @deprecated
- */
- @Deprecated
- public Map<String, String> properties() {
- return props;
- }
+// /**
+// * @return Secondary file system properties.
+// *
+// * @deprecated
+// */
+// @Deprecated
+// public Map<String, String> properties() {
+// return props;
+// }
/**
* @return Default IGFS mode.
@@ -114,9 +113,9 @@ public class IgfsPaths <F> implements Externalizable {
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
- U.writeStringMap(out, props);
+// U.writeStringMap(out, props);
-// writeFactory(out);
+ writePayload(out);
U.writeEnum(out, dfltMode);
@@ -133,30 +132,31 @@ public class IgfsPaths <F> implements Externalizable {
out.writeBoolean(false);
}
-// /**
-// *
-// * @param out
-// * @throws IOException
-// */
-// private void writeFactory(ObjectOutput out) throws IOException {
-// ByteArrayOutputStream baos = new ByteArrayOutputStream();
-//
-// ObjectOutput oo = new ObjectOutputStream(baos);
-// try {
-// oo.writeObject(factory);
-// }
-// finally {
-// oo.close();
-// }
-//
-// U.writeByteArray(out, baos.toByteArray());
-// }
+ /**
+ *
+ * @param out
+ * @throws IOException
+ */
+ private void writePayload(ObjectOutput out) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ ObjectOutput oo = new ObjectOutputStream(baos);
+
+ try {
+ oo.writeObject(payload);
+ }
+ finally {
+ oo.close();
+ }
+
+ U.writeByteArray(out, baos.toByteArray());
+ }
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- props = U.readStringMap(in);
+// props = U.readStringMap(in);
-// readFactory(in);
+ readPayload(in);
dfltMode = IgfsMode.fromOrdinal(in.readByte());
@@ -176,22 +176,26 @@ public class IgfsPaths <F> implements Externalizable {
}
}
-// /**
-// *
-// * @param in
-// * @throws IOException
-// * @throws ClassNotFoundException
-// */
-// private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
-// byte[] factoryBytes = U.readByteArray(in);
-//
-// ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
-//
-// try {
-// factory = (HadoopFileSystemFactory<F>) oi.readObject();
-// }
-// finally {
-// oi.close();
-// }
-// }
+ /**
+ *
+ * @param in
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ private void readPayload(ObjectInput in) throws IOException, ClassNotFoundException {
+ byte[] factoryBytes = U.readByteArray(in);
+
+ ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+
+ try {
+ payload = (P) oi.readObject();
+ }
+ finally {
+ oi.close();
+ }
+ }
+
+ @Override public P getPayload() {
+ return payload;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
index 1b1ce24..ced6b21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
@@ -19,10 +19,8 @@ package org.apache.ignite.internal.processors.igfs;
import java.io.OutputStream;
import java.util.Collection;
-import java.util.Collections;
import java.util.Map;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
@@ -32,7 +30,7 @@ import org.jetbrains.annotations.Nullable;
/**
* Secondary file system over native IGFS.
*/
-class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem <IgfsEx> {
+class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
/** Delegate. */
private final IgfsEx igfs;
@@ -118,18 +116,18 @@ class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem <IgfsEx> {
return igfs.usedSpaceSize();
}
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
+// /** {@inheritDoc} */
+// @Override public Map<String, String> properties() {
+// return Collections.emptyMap();
+// }
/** {@inheritDoc} */
@Override public void close() throws IgniteException {
// No-op.
}
- /** {@inheritDoc} */
- @Override public HadoopFileSystemFactory<IgfsEx> getSecondaryFileSystemFactory() {
- return null;
- }
+// /** {@inheritDoc} */
+// @Override public HadoopFileSystemFactory<IgfsEx> getSecondaryFileSystemFactory() {
+// return null;
+// }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 08792a4..2238fdf 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Map;
import org.apache.ignite.configuration.FileSystemConfiguration;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsIpcEndpointConfiguration;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index f8c7f3a..7ba136b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -21,7 +21,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
-import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -36,7 +35,6 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -47,6 +45,7 @@ import org.apache.ignite.igfs.IgfsPathNotFoundException;
import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
+import org.apache.ignite.internal.processors.hadoop.PayloadAware;
import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable;
@@ -57,7 +56,6 @@ import org.apache.ignite.internal.util.typedef.F;
import static org.apache.ignite.internal.util.typedef.F.*;
import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
@@ -70,7 +68,8 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
* In fact, this class deals with different FileSystems depending on the user context,
* see {@link IgfsUserContext#currentUser()}.
*/
-public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem <FileSystem> {
+public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem,
+ LifecycleAware, PayloadAware<HadoopFileSystemFactory<FileSystem>> {
// /** Properties of file system, see {@link #properties()}
// * */
// private final Map<String, String> props = new HashMap<>();
@@ -533,10 +532,10 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
}
}
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
+// /** {@inheritDoc} */
+// @Override public Map<String, String> properties() {
+// return Collections.emptyMap();
+// }
/** {@inheritDoc} */
@Override public void close() throws IgniteException {
@@ -589,7 +588,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
if (F.eq(user, dfltUserName))
return dfltFs; // optimization
- assert fsFactory.uri() != null : "uri!";
+ //assert fsFactory.uri() != null : "uri!";
try {
return fsFactory.get(user);
@@ -605,7 +604,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
*
* @throws IgniteCheckedException
*/
- public void start() throws IgniteCheckedException {
+ @Override public void start() {
// #start() should not ever be invoked if these properties are not set:
A.ensure(fsFactory != null, "factory");
A.ensure(dfltUserName != null, "dfltUserName");
@@ -625,13 +624,21 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
assert dfltFs != null;
}
catch (IOException e) {
- throw new IgniteCheckedException(e);
+ throw new IgniteException(e);
}
}
}
- /** {@inheritDoc} */
- @Nullable @Override public HadoopFileSystemFactory<FileSystem> getSecondaryFileSystemFactory() {
+// /** {@inheritDoc} */
+// @Nullable @Override public HadoopFileSystemFactory<FileSystem> getSecondaryFileSystemFactory() {
+// return fsFactory;
+// }
+
+ @Override public void stop() throws IgniteException {
+ close();
+ }
+
+ @Override public HadoopFileSystemFactory<FileSystem> getPayload() {
return fsFactory;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 20e2011..932e326 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
+import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -295,7 +295,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
igfsGrpBlockSize = handshake.blockSize();
- final IgfsPaths<FileSystem> paths = handshake.secondaryPaths();
+ final IgfsPaths<HadoopFileSystemFactory<FileSystem>> paths = handshake.secondaryPaths();
// Initialize client logger.
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -334,26 +334,24 @@ public class IgniteHadoopFileSystem extends FileSystem {
// String secUri = props.get(SECONDARY_FS_URI);
// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- byte[] secFsFacoryBytes = handshake.getSecondaryFileSystemFactoryBytes();
+// byte[] secFsFacoryBytes = handshake.getSecondaryFileSystemFactoryBytes();
-
-
- //HadoopFileSystemFactory<FileSystem> factory = paths.factory();
+ HadoopFileSystemFactory<FileSystem> factory = paths.getPayload();
A.ensure(factory != null, "Secondary file system factory should not be null.");
- secondaryUri = factory.uri();
-
- A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
+ //secondaryUri = factory.uri();
try {
//SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
secondaryFs = factory.get(user); //secProvider.createFileSystem(user);
- URI uri2 = secondaryFs.getUri();
+ secondaryUri = secondaryFs.getUri();
+
+ A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
- assert secondaryUri.equals(uri2);
+ //assert secondaryUri.equals(uri2);
}
catch (IOException e) {
if (!mgmt)
@@ -372,23 +370,22 @@ public class IgniteHadoopFileSystem extends FileSystem {
}
}
- /**
- *
- * @param in
- * @throws IOException
- * @throws ClassNotFoundException
- */
- static HadoopFileSystemFactory readFactory(byte[] factoryBytes) throws IOException, ClassNotFoundException {
- ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
-
- try {
- return (HadoopFileSystemFactory<F>) oi.readObject();
- }
- finally {
- oi.close();
- }
- }
-
+// /**
+// *
+// * @param in
+// * @throws IOException
+// * @throws ClassNotFoundException
+// */
+// static HadoopFileSystemFactory readFactory(byte[] factoryBytes) throws IOException, ClassNotFoundException {
+// ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+//
+// try {
+// return (HadoopFileSystemFactory<F>) oi.readObject();
+// }
+// finally {
+// oi.close();
+// }
+// }
/** {@inheritDoc} */
@Override protected void checkPath(Path path) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index ae97464..d3267c7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -52,13 +52,12 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
+import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.igfs.common.IgfsLogger;
-import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutputStream;
@@ -93,8 +92,6 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PREFER_LOCAL_WRITES;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
-//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_CONFIG_PATH;
-//import static org.apache.ignite.internal.processors.igfs.IgfsEx.SECONDARY_FS_URI;
/**
* {@code IGFS} Hadoop 2.x file system driver over file system API. To use
@@ -303,7 +300,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
grpBlockSize = handshake.blockSize();
- IgfsPaths<AbstractFileSystem> paths = handshake.secondaryPaths();
+ IgfsPaths<HadoopFileSystemFactory<AbstractFileSystem>> paths = handshake.secondaryPaths();
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -342,7 +339,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
HadoopFileSystemFactory<AbstractFileSystem> factory
- = (HadoopFileSystemFactory<AbstractFileSystem>)paths.factory();
+ = (HadoopFileSystemFactory<AbstractFileSystem>)paths.getPayload();
A.ensure(secondaryUri != null, "File system factory uri should not be null.");
@@ -355,10 +352,10 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
secondaryUri = secondaryFs.getUri();
- assert secondaryUri != null;
-
- URI uri2 = ((DefaultHadoopFileSystemFactory)factory).uri();
- assert secondaryUri.equals(uri2);
+// assert secondaryUri != null;
+//
+// URI uri2 = ((DefaultHadoopFileSystemFactory)factory).uri();
+// assert secondaryUri.equals(uri2);
//secondaryFs = secProvider.createAbstractFileSystem(user);
//secondaryUri = secProvider.uri();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d12b9f12/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
index 0b93cba..bee0f25 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
@@ -12,7 +12,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
+import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
import org.apache.ignite.internal.processors.igfs.IgfsPaths;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
@@ -75,10 +75,6 @@ public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<F
}
}
- @Override public URI uri() {
- return uri;
- }
-
/**
* Configuration(s) setter, to be invoked from Spring config.
* @param cfgPaths
[02/12] ignite git commit: IGNITE-2206: intermediate more or less
working state, before review corrections.
Posted by vo...@apache.org.
IGNITE-2206: intermediate more or less working state, before review corrections.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/31c40d68
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/31c40d68
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/31c40d68
Branch: refs/heads/ignite-2206
Commit: 31c40d6857f0c886638e0008dd4ea3b6bedeea61
Parents: 7a3f9dd
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 16:05:48 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 16:05:48 2015 +0300
----------------------------------------------------------------------
.../internal/processors/igfs/IgfsPaths.java | 46 ++++++++++++++-
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 59 ++++++++++++++------
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 2 +
.../fs/DefaultHadoopFileSystemFactory.java | 17 ++++--
.../IgniteHadoopFileSystemAbstractSelfTest.java | 20 ++++++-
...teHadoopFileSystemShmemAbstractSelfTest.java | 2 +
6 files changed, 121 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index bf7e825..809c7da 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -17,10 +17,14 @@
package org.apache.ignite.internal.processors.igfs;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.Externalizable;
import java.io.IOException;
import java.io.ObjectInput;
+import java.io.ObjectInputStream;
import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -112,7 +116,7 @@ public class IgfsPaths <F> implements Externalizable {
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeStringMap(out, props);
- out.writeObject(factory);
+ writeFactory(out);
U.writeEnum(out, dfltMode);
@@ -129,11 +133,30 @@ public class IgfsPaths <F> implements Externalizable {
out.writeBoolean(false);
}
+ /**
+ *
+ * @param out
+ * @throws IOException
+ */
+ private void writeFactory(ObjectOutput out) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ ObjectOutput oo = new ObjectOutputStream(baos);
+ try {
+ oo.writeObject(factory);
+ }
+ finally {
+ oo.close();
+ }
+
+ U.writeByteArray(out, baos.toByteArray());
+ }
+
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
props = U.readStringMap(in);
- factory = (HadoopFileSystemFactory<F>)in.readObject();
+ readFactory(in);
dfltMode = IgfsMode.fromOrdinal(in.readByte());
@@ -152,4 +175,23 @@ public class IgfsPaths <F> implements Externalizable {
}
}
}
+
+ /**
+ *
+ * @param in
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
+ byte[] factoryBytes = U.readByteArray(in);
+
+ ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+
+ try {
+ factory = (HadoopFileSystemFactory<F>) oi.readObject();
+ }
+ finally {
+ oi.close();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index c5124f0..f8c7f3a 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -21,10 +21,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URI;
+import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.ParentNotDirectoryException;
@@ -45,6 +47,7 @@ import org.apache.ignite.igfs.IgfsPathNotFoundException;
import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
+import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable;
import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
@@ -97,6 +100,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** */
private HadoopFileSystemFactory<FileSystem> fsFactory;
+ private final AtomicBoolean started = new AtomicBoolean();
+
// /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
// private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
// new ValueFactory<String, FileSystem>() {
@@ -149,26 +154,41 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
* @param cfgPath Additional path to Hadoop configuration.
* @param userName User name.
* @throws IgniteCheckedException In case of error.
+ * @deprecated Arg-less constructor should be used instead, + setters. This constructor is
+ * supported for compatibility only.
*/
+ @Deprecated
public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath,
@Nullable String userName) throws IgniteCheckedException {
uri = nullifyEmpty(uri);
- if (uri != null)
- U.warn(null, "This constructor is deprecated. URI value passed in will be ignored.");
+// if (uri != null)
+// U.warn(null, "This constructor is deprecated. URI value passed in will be ignored.");
cfgPath = nullifyEmpty(cfgPath);
+// if (cfgPath != null)
+// U.warn(null, "This constructor is deprecated. The configurationPath value passed in will be ignored.");
+
+ DefaultHadoopFileSystemFactory fac = new DefaultHadoopFileSystemFactory();
+
+ if (uri != null)
+ fac.setUri(uri);
+
if (cfgPath != null)
- U.warn(null, "This constructor is deprecated. The configurationPath value passed in will be ignored.");
+ fac.setCfgPaths(Collections.singletonList(cfgPath));
+
+ setFsFactory(fac);
setDfltUserName(userName);
+
+ start();
}
/**
*
* @param factory
*/
- public void setFsFactory(HadoopFileSystemFactory factory) {
+ public void setFsFactory(HadoopFileSystemFactory<FileSystem> factory) {
A.ensure(factory != null, "Factory value must not be null.");
this.fsFactory = factory;
@@ -557,6 +577,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
* @return the FileSystem instance, never null.
*/
private FileSystem fileSysForUser() {
+ assert started.get(); // Ensure the Fs is started.
+
String user = IgfsUserContext.currentUser();
if (F.isEmpty(user))
@@ -567,6 +589,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
if (F.eq(user, dfltUserName))
return dfltFs; // optimization
+ assert fsFactory.uri() != null : "uri!";
+
try {
return fsFactory.get(user);
}
@@ -582,24 +606,27 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
* @throws IgniteCheckedException
*/
public void start() throws IgniteCheckedException {
+ // #start() should not ever be invoked if these properties are not set:
A.ensure(fsFactory != null, "factory");
A.ensure(dfltUserName != null, "dfltUserName");
- if (fsFactory instanceof LifecycleAware)
- ((LifecycleAware) fsFactory).start();
+ if (started.compareAndSet(false, true)) {
+ if (fsFactory instanceof LifecycleAware)
+ ((LifecycleAware) fsFactory).start();
- try {
- //this.secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
+ try {
+ //this.secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
- // File system creation for the default user name.
- // The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
- //this.dfltFs = secProvider.createFileSystem(dfltUserName);
- this.dfltFs = fsFactory.get(dfltUserName);
+ // File system creation for the default user name.
+ // The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
+ //this.dfltFs = secProvider.createFileSystem(dfltUserName);
+ this.dfltFs = fsFactory.get(dfltUserName);
- assert dfltFs != null;
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
+ assert dfltFs != null;
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 1b748fb..2fcf774 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -258,6 +258,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
"://[name]/[optional_path], actual=" + name + ']');
uri = name;
+ System.out.println("uri initialized: " + uri);
uriAuthority = uri.getAuthority();
@@ -418,6 +419,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
// Reset initialized resources.
uri = null;
+ System.out.println("uri zeroed.");
rmtClient = null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
index 5bbf4d9..246637d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
@@ -62,14 +62,23 @@ public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<F
this.uri = uri;
}
+ /**
+ * Convenience mathod, analog of {@link #setUri(URI)} with String type argument.
+ * @param uriStr
+ */
+ public void setUri(String uriStr) {
+ try {
+ setUri(new URI(uriStr));
+ }
+ catch (URISyntaxException use) {
+ throw new IgniteException(use);
+ }
+ }
+
@Override public URI uri() {
return uri;
}
-// public void setCfg(Configuration cfg) {
-// this.cfg = cfg;
-// }
-
/**
* Configuration(s) setter, to be invoked from Spring config.
* @param cfgPaths
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index d368955..7e5ef39 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -25,10 +25,12 @@ import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.URI;
+import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayDeque;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Deque;
import java.util.LinkedList;
@@ -61,6 +63,7 @@ import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
+import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEx;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsIpcIo;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutProc;
@@ -380,9 +383,20 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
cfg.setPrefetchBlocks(1);
cfg.setDefaultMode(mode);
- if (mode != PRIMARY)
- cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
- SECONDARY_URI, SECONDARY_CFG_PATH, SECONDARY_FS_USER));
+ if (mode != PRIMARY) {
+ DefaultHadoopFileSystemFactory fac = new DefaultHadoopFileSystemFactory();
+ fac.setUri(SECONDARY_URI);
+ fac.setCfgPaths(Collections.singletonList(SECONDARY_CFG_PATH));
+
+ IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem();
+
+ sec.setFsFactory(fac);
+ sec.setDfltUserName(SECONDARY_FS_USER);
+
+ sec.start();
+
+ cfg.setSecondaryFileSystem(sec);
+ }
cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
http://git-wip-us.apache.org/repos/asf/ignite/blob/31c40d68/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
index d8cf74c..20c2bd2 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
@@ -60,6 +60,8 @@ public abstract class IgniteHadoopFileSystemShmemAbstractSelfTest extends Ignite
*/
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
public void testOutOfResources() throws Exception {
+ if (1 == 1) return;
+
final Collection<IpcEndpoint> eps = new LinkedList<>();
try {
[05/12] ignite git commit: IGNITE-2206: intermadiate for review.
Posted by vo...@apache.org.
IGNITE-2206: intermadiate for review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/4553562b
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/4553562b
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/4553562b
Branch: refs/heads/ignite-2206
Commit: 4553562b99b345b086ed339b47b6ae8b95f08c0c
Parents: d485969
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 18:17:56 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 18:17:56 2015 +0300
----------------------------------------------------------------------
.../ignite/igfs/HadoopFileSystemFactory.java | 29 --------------------
.../processors/hadoop/PayloadAware.java | 9 ++++++
.../hadoop/fs/HadoopFileSystemFactory.java | 29 ++++++++++++++++++++
3 files changed, 38 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/4553562b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
deleted file mode 100644
index 6720da9..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.ignite.igfs;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-/**
- * This factory is {@link Serializable} because it should be transferable over the network.
- *
- * @param <T> The type
- */
-public interface HadoopFileSystemFactory <T> extends Serializable {
- /**
- * Gets the file system, possibly creating it or taking a cached instance.
- * All the other data needed for the file system creation are expected to be contained
- * in this object instance.
- *
- * @param userName The user name
- * @return The file system.
- * @throws IOException On error.
- */
- public T get(String userName) throws IOException;
-
-// /**
-// * Getter for the file system URI.
-// *
-// * @return The file system URI.
-// */
-// public URI uri();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4553562b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
new file mode 100644
index 0000000..24a8545
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
@@ -0,0 +1,9 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ * Created by ivan on 22.12.15.
+ */
+public interface PayloadAware <P> {
+
+ public P getPayload();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/4553562b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
new file mode 100644
index 0000000..5337f12
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
@@ -0,0 +1,29 @@
+package org.apache.ignite.hadoop.fs;
+
+import java.io.IOException;
+import java.io.Serializable;
+
+/**
+ * This factory is {@link Serializable} because it should be transferable over the network.
+ *
+ * @param <T> The type
+ */
+public interface HadoopFileSystemFactory <T> extends Serializable {
+ /**
+ * Gets the file system, possibly creating it or taking a cached instance.
+ * All the other data needed for the file system creation are expected to be contained
+ * in this object instance.
+ *
+ * @param userName The user name
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public T get(String userName) throws IOException;
+
+// /**
+// * Getter for the file system URI.
+// *
+// * @return The file system URI.
+// */
+// public URI uri();
+}
[08/12] ignite git commit: IGNITE-2206: intermediate saving commit.
Posted by vo...@apache.org.
IGNITE-2206: intermediate saving commit.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/89690022
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/89690022
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/89690022
Branch: refs/heads/ignite-2206
Commit: 89690022863a32fb00e76335c02c6a36bfae55e1
Parents: ba11b3a
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 22:51:07 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 22:51:07 2015 +0300
----------------------------------------------------------------------
.../processors/hadoop/HadoopPayloadAware.java | 12 ++
.../processors/hadoop/PayloadAware.java | 13 --
.../internal/processors/igfs/IgfsImpl.java | 12 +-
.../internal/processors/igfs/IgfsPaths.java | 37 +---
.../hadoop/fs/HadoopFileSystemFactory.java | 29 ---
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 19 +-
.../fs/v1/DefaultHadoopFileSystemFactory.java | 185 ++++++++++++++++++
.../hadoop/fs/v1/HadoopFileSystemFactory.java | 21 +++
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 27 ++-
.../fs/v2/HadoopAbstractFileSystemFactory.java | 21 +++
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 7 +-
.../fs/DefaultHadoopFileSystemFactory.java | 187 -------------------
.../hadoop/fs/HadoopLazyConcurrentMap.java | 4 +
.../IgniteHadoopFileSystemAbstractSelfTest.java | 12 +-
...teHadoopFileSystemShmemAbstractSelfTest.java | 2 -
15 files changed, 284 insertions(+), 304 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
new file mode 100644
index 0000000..dcb163f
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/HadoopPayloadAware.java
@@ -0,0 +1,12 @@
+package org.apache.ignite.internal.processors.hadoop;
+
+/**
+ *
+ */
+public interface HadoopPayloadAware {
+ /**
+ *
+ * @return
+ */
+ public Object getPayload();
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
deleted file mode 100644
index dc4ff5e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/hadoop/PayloadAware.java
+++ /dev/null
@@ -1,13 +0,0 @@
-package org.apache.ignite.internal.processors.hadoop;
-
-/**
- * Created by ivan on 22.12.15.
- */
-public interface PayloadAware <P> {
-
- /**
- *
- * @return
- */
- public P getPayload();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index fb93ea1..7453e15 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -19,7 +19,6 @@ package org.apache.ignite.internal.processors.igfs;
import java.io.IOException;
import java.io.OutputStream;
-import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
@@ -73,7 +72,7 @@ import org.apache.ignite.internal.IgniteKernal;
import org.apache.ignite.internal.managers.communication.GridMessageListener;
import org.apache.ignite.internal.managers.eventstorage.GridEventStorageManager;
import org.apache.ignite.internal.managers.eventstorage.GridLocalEventListener;
-import org.apache.ignite.internal.processors.hadoop.PayloadAware;
+import org.apache.ignite.internal.processors.hadoop.HadoopPayloadAware;
import org.apache.ignite.internal.processors.task.GridInternal;
import org.apache.ignite.internal.util.GridSpinBusyLock;
import org.apache.ignite.internal.util.future.GridCompoundFuture;
@@ -125,7 +124,7 @@ public final class IgfsImpl implements IgfsEx {
static final Map<String, String> DFLT_DIR_META = F.asMap(PROP_PERMISSION, PERMISSION_DFLT_VAL);
/** Handshake message. */
- private final IgfsPaths<Serializable> secondaryPaths;
+ private final IgfsPaths secondaryPaths;
/** Cache based structure (meta data) manager. */
private IgfsMetaManager meta;
@@ -260,11 +259,10 @@ public final class IgfsImpl implements IgfsEx {
modeRslvr = new IgfsModeResolver(dfltMode, modes);
- Serializable secondaryFsPayload = null;
+ Object secondaryFsPayload = null;
- if (secondaryFs instanceof PayloadAware) {
- secondaryFsPayload = ((PayloadAware<Serializable>) secondaryFs).getPayload();
- }
+ if (secondaryFs instanceof HadoopPayloadAware)
+ secondaryFsPayload = ((HadoopPayloadAware) secondaryFs).getPayload();
secondaryPaths = new IgfsPaths(
//secondaryFs == null ? null : secondaryFs.properties(),
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index d434d01..83451db 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -25,12 +25,10 @@ import java.io.ObjectInput;
import java.io.ObjectInputStream;
import java.io.ObjectOutput;
import java.io.ObjectOutputStream;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
-import org.apache.ignite.internal.processors.hadoop.PayloadAware;
import org.apache.ignite.internal.util.typedef.T2;
import org.apache.ignite.internal.util.typedef.internal.U;
import org.jetbrains.annotations.Nullable;
@@ -38,16 +36,12 @@ import org.jetbrains.annotations.Nullable;
/**
* Description of path modes.
*/
-public class IgfsPaths <P extends Serializable> implements Externalizable, PayloadAware<P> {
+public class IgfsPaths implements Externalizable {
/** */
private static final long serialVersionUID = 0L;
-// /** Additional secondary file system properties. */
-// @Deprecated
-// private Map<String, String> props;
-
/** */
- private P payload;
+ private Object payload;
/** Default IGFS mode. */
private IgfsMode dfltMode;
@@ -68,26 +62,14 @@ public class IgfsPaths <P extends Serializable> implements Externalizable, Paylo
* @param dfltMode Default IGFS mode.
* @param pathModes Path modes.
*/
- public IgfsPaths(//Map<String, String> props,
- P payload,
+ public IgfsPaths(Object payload,
IgfsMode dfltMode,
@Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
- //this.props = props;
this.payload = payload;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
}
-// /**
-// * @return Secondary file system properties.
-// *
-// * @deprecated
-// */
-// @Deprecated
-// public Map<String, String> properties() {
-// return props;
-// }
-
/**
* @return Default IGFS mode.
*/
@@ -102,15 +84,6 @@ public class IgfsPaths <P extends Serializable> implements Externalizable, Paylo
return pathModes;
}
-// /**
-// * Getter for factory.
-// *
-// * @return The factory.
-// */
-// public HadoopFileSystemFactory<F> factory() {
-// return factory;
-// }
-
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
// U.writeStringMap(out, props);
@@ -188,14 +161,14 @@ public class IgfsPaths <P extends Serializable> implements Externalizable, Paylo
ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
try {
- payload = (P) oi.readObject();
+ payload = oi.readObject();
}
finally {
oi.close();
}
}
- @Override public P getPayload() {
+ public Object getPayload() {
return payload;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
deleted file mode 100644
index 5337f12..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/HadoopFileSystemFactory.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.apache.ignite.hadoop.fs;
-
-import java.io.IOException;
-import java.io.Serializable;
-
-/**
- * This factory is {@link Serializable} because it should be transferable over the network.
- *
- * @param <T> The type
- */
-public interface HadoopFileSystemFactory <T> extends Serializable {
- /**
- * Gets the file system, possibly creating it or taking a cached instance.
- * All the other data needed for the file system creation are expected to be contained
- * in this object instance.
- *
- * @param userName The user name
- * @return The file system.
- * @throws IOException On error.
- */
- public T get(String userName) throws IOException;
-
-// /**
-// * Getter for the file system URI.
-// *
-// * @return The file system URI.
-// */
-// public URI uri();
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 7ba136b..86ed7a0 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.fs.PathIsNotEmptyDirectoryException;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.hadoop.fs.v1.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsDirectoryNotEmptyException;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -45,8 +46,8 @@ import org.apache.ignite.igfs.IgfsPathNotFoundException;
import org.apache.ignite.igfs.IgfsUserContext;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystemPositionedReadable;
-import org.apache.ignite.internal.processors.hadoop.PayloadAware;
-import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
+import org.apache.ignite.internal.processors.hadoop.HadoopPayloadAware;
+import org.apache.ignite.hadoop.fs.v1.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsProperties;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsSecondaryFileSystemPositionedReadable;
import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
@@ -69,7 +70,7 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
* see {@link IgfsUserContext#currentUser()}.
*/
public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem,
- LifecycleAware, PayloadAware<HadoopFileSystemFactory<FileSystem>> {
+ LifecycleAware, HadoopPayloadAware {
// /** Properties of file system, see {@link #properties()}
// * */
// private final Map<String, String> props = new HashMap<>();
@@ -97,7 +98,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
private String dfltUserName = IgfsUtils.fixUserName(null);
/** */
- private HadoopFileSystemFactory<FileSystem> fsFactory;
+ private HadoopFileSystemFactory fsFactory;
private final AtomicBoolean started = new AtomicBoolean();
@@ -174,20 +175,18 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
fac.setUri(uri);
if (cfgPath != null)
- fac.setCfgPaths(Collections.singletonList(cfgPath));
+ fac.setConfigPaths(Collections.singletonList(cfgPath));
setFsFactory(fac);
setDfltUserName(userName);
-
- start();
}
/**
*
* @param factory
*/
- public void setFsFactory(HadoopFileSystemFactory<FileSystem> factory) {
+ public void setFsFactory(HadoopFileSystemFactory factory) {
A.ensure(factory != null, "Factory value must not be null.");
this.fsFactory = factory;
@@ -600,7 +599,6 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/**
* Should be invoked by client (from Spring?) after all the setters invoked.
- * TODO: how this should be invoked?
*
* @throws IgniteCheckedException
*/
@@ -609,6 +607,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
A.ensure(fsFactory != null, "factory");
A.ensure(dfltUserName != null, "dfltUserName");
+ // Avoid
if (started.compareAndSet(false, true)) {
if (fsFactory instanceof LifecycleAware)
((LifecycleAware) fsFactory).start();
@@ -638,7 +637,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
close();
}
- @Override public HadoopFileSystemFactory<FileSystem> getPayload() {
+ @Override public HadoopFileSystemFactory getPayload() {
return fsFactory;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
new file mode 100644
index 0000000..ba4bfdd
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/DefaultHadoopFileSystemFactory.java
@@ -0,0 +1,185 @@
+package org.apache.ignite.hadoop.fs.v1;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
+import org.apache.ignite.internal.processors.hadoop.fs.HadoopFileSystemsUtils;
+import org.apache.ignite.internal.processors.hadoop.fs.HadoopLazyConcurrentMap;
+import org.apache.ignite.internal.processors.igfs.IgfsPaths;
+import org.apache.ignite.internal.processors.igfs.IgfsUtils;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.A;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lifecycle.LifecycleAware;
+
+import static org.apache.ignite.internal.util.lang.GridFunc.nullifyEmpty;
+
+/**
+ * The class is to be instantiated as a Spring beans, so it must have public zero-arg constructor.
+ * The class is serializable as it will be transferred over the network as a part of {@link IgfsPaths} object.
+ */
+public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory, Externalizable, LifecycleAware {
+ /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
+ private final transient HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
+ new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
+ @Override public FileSystem createValue(String key) {
+ try {
+ assert !F.isEmpty(key);
+
+ return createFileSystem(key);
+ }
+ catch (IOException ioe) {
+ throw new IgniteException(ioe);
+ }
+ }
+ }
+ );
+
+ /** Configuration of the secondary filesystem, never null. */
+ protected transient Configuration cfg;
+
+ /** */
+ protected transient URI uri;
+
+ /** */
+ protected String uriStr;
+
+ /** */
+ protected List<String> cfgPathStr;
+
+ int getCount = 0;
+
+ /**
+ *
+ */
+ public DefaultHadoopFileSystemFactory() {
+ //
+
+
+
+ }
+
+ @Override public FileSystem get(String userName) throws IOException {
+ A.ensure(cfg != null, "cfg");
+
+ if (getCount == 0)
+ assert fileSysLazyMap.size() == 0;
+
+ getCount++;
+
+ return fileSysLazyMap.getOrCreate(userName);
+ }
+
+ /**
+ * Uri setter.
+ * @param uriStr
+ */
+ public void setUri(String uriStr) {
+ this.uriStr = uriStr;
+ }
+
+ /**
+ * Configuration(s) setter, to be invoked from Spring config.
+ * @param cfgPaths
+ */
+ public void setConfigPaths(List<String> cfgPaths) {
+ this.cfgPathStr = (List)nullifyEmpty(cfgPaths);
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ protected FileSystem createFileSystem(String userName) throws IOException {
+ userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
+
+ assert cfg != null;
+
+ final FileSystem fileSys;
+
+ try {
+ fileSys = FileSystem.get(uri, cfg, userName);
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+
+ throw new IOException("Failed to create file system due to interrupt.", e);
+ }
+
+ return fileSys;
+ }
+
+ @Override public void writeExternal(ObjectOutput out) throws IOException {
+ U.writeString(out, uriStr);
+
+ U.writeCollection(out, cfgPathStr);
+ }
+
+ @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ uriStr = U.readString(in);
+
+ cfgPathStr = new ArrayList(U.readCollection(in));
+ }
+
+ @Override public void start() throws IgniteException {
+ cfg = HadoopUtils.safeCreateConfiguration();
+
+ if (cfgPathStr != null) {
+ for (String confPath : cfgPathStr) {
+ confPath = nullifyEmpty(confPath);
+
+ if (confPath != null) {
+ URL url = U.resolveIgniteUrl(confPath);
+
+ if (url == null) {
+ // If secConfPath is given, it should be resolvable:
+ throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
+
+ "(ensure that it exists locally and you have read access to it): " + confPath);
+ }
+
+ cfg.addResource(url);
+ }
+ }
+ }
+
+ // if secondary fs URI is not given explicitly, try to get it from the configuration:
+ if (uriStr == null)
+ uri = FileSystem.getDefaultUri(cfg);
+ else {
+ try {
+ uri = new URI(uriStr);
+ }
+ catch (URISyntaxException use) {
+ throw new IgniteException("Failed to resolve secondary file system URI: " + uriStr);
+ }
+ }
+
+ assert uriStr != null;
+
+ // Disable caching:
+ String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(uri.getScheme());
+
+ cfg.setBoolean(prop, true);
+ }
+
+ @Override public void stop() throws IgniteException {
+ try {
+ fileSysLazyMap.close();
+ }
+ catch (IgniteCheckedException ice) {
+ throw new IgniteException(ice);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
new file mode 100644
index 0000000..c1c7b9d
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/HadoopFileSystemFactory.java
@@ -0,0 +1,21 @@
+package org.apache.ignite.hadoop.fs.v1;
+
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.fs.FileSystem;
+
+/**
+ * This factory is {@link Serializable} because it should be transferable over the network.
+ */
+public interface HadoopFileSystemFactory extends Serializable {
+ /**
+ * Gets the file system, possibly creating it or taking a cached instance.
+ * All the other data needed for the file system creation are expected to be contained
+ * in this object instance.
+ *
+ * @param userName The user name
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public FileSystem get(String userName) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 932e326..355892e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -18,12 +18,9 @@
package org.apache.ignite.hadoop.fs.v1;
import java.io.BufferedOutputStream;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
@@ -47,7 +44,6 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsException;
import org.apache.ignite.igfs.IgfsFile;
@@ -71,6 +67,7 @@ import org.apache.ignite.internal.util.typedef.X;
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.internal.util.typedef.internal.S;
import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
import static org.apache.ignite.configuration.FileSystemConfiguration.DFLT_IGFS_LOG_BATCH_SIZE;
@@ -190,7 +187,8 @@ public class IgniteHadoopFileSystem extends FileSystem {
/** {@inheritDoc} */
@Override public URI getUri() {
if (uri == null)
- throw new IllegalStateException("URI is null (was IgniteHadoopFileSystem properly initialized?).");
+ throw new IllegalStateException("URI is null (was IgniteHadoopFileSystem properly initialized?) [closed="
+ + closeGuard.get() + ']');
return uri;
}
@@ -243,6 +241,8 @@ public class IgniteHadoopFileSystem extends FileSystem {
@Override public void initialize(URI name, Configuration cfg) throws IOException {
enterBusy();
+ assert !closeGuard.get();
+
try {
if (rmtClient != null)
throw new IOException("File system is already initialized: " + rmtClient);
@@ -295,7 +295,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
igfsGrpBlockSize = handshake.blockSize();
- final IgfsPaths<HadoopFileSystemFactory<FileSystem>> paths = handshake.secondaryPaths();
+ final IgfsPaths paths = handshake.secondaryPaths();
// Initialize client logger.
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -336,16 +336,15 @@ public class IgniteHadoopFileSystem extends FileSystem {
// byte[] secFsFacoryBytes = handshake.getSecondaryFileSystemFactoryBytes();
- HadoopFileSystemFactory<FileSystem> factory = paths.getPayload();
+ HadoopFileSystemFactory factory = (HadoopFileSystemFactory)paths.getPayload();
A.ensure(factory != null, "Secondary file system factory should not be null.");
- //secondaryUri = factory.uri();
+ if (factory instanceof LifecycleAware)
+ ((LifecycleAware) factory).start();
try {
- //SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
-
- secondaryFs = factory.get(user); //secProvider.createFileSystem(user);
+ secondaryFs = factory.get(user);
secondaryUri = secondaryFs.getUri();
@@ -440,12 +439,12 @@ public class IgniteHadoopFileSystem extends FileSystem {
if (clientLog.isLogEnabled())
clientLog.close();
- if (secondaryFs != null)
- U.closeQuiet(secondaryFs);
+ U.closeQuiet(secondaryFs);
+
+ System.out.println("closed " + uri);
// Reset initialized resources.
uri = null;
- System.out.println("uri zeroed.");
rmtClient = null;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopAbstractFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopAbstractFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopAbstractFileSystemFactory.java
new file mode 100644
index 0000000..cf81e57
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/HadoopAbstractFileSystemFactory.java
@@ -0,0 +1,21 @@
+package org.apache.ignite.hadoop.fs.v2;
+
+import java.io.IOException;
+import java.io.Serializable;
+import org.apache.hadoop.fs.AbstractFileSystem;
+
+/**
+ * This factory is {@link Serializable} because it should be transferable over the network.
+ */
+interface HadoopAbstractFileSystemFactory extends Serializable {
+ /**
+ * Gets the file system, possibly creating it or taking a cached instance.
+ * All the other data needed for the file system creation are expected to be contained
+ * in this object instance.
+ *
+ * @param userName The user name
+ * @return The file system.
+ * @throws IOException On error.
+ */
+ public AbstractFileSystem get(String userName) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index d3267c7..96f97dc 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -52,7 +52,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Progressable;
import org.apache.ignite.IgniteException;
-import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsBlockLocation;
import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
@@ -300,7 +299,7 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
grpBlockSize = handshake.blockSize();
- IgfsPaths<HadoopFileSystemFactory<AbstractFileSystem>> paths = handshake.secondaryPaths();
+ IgfsPaths paths = handshake.secondaryPaths();
Boolean logEnabled = parameter(cfg, PARAM_IGFS_LOG_ENABLED, uriAuthority, false);
@@ -338,8 +337,8 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
// String secUri = props.get(SECONDARY_FS_URI);
// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- HadoopFileSystemFactory<AbstractFileSystem> factory
- = (HadoopFileSystemFactory<AbstractFileSystem>)paths.getPayload();
+ HadoopAbstractFileSystemFactory factory
+ = (HadoopAbstractFileSystemFactory)paths.getPayload();
A.ensure(secondaryUri != null, "File system factory uri should not be null.");
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
deleted file mode 100644
index bee0f25..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
+++ /dev/null
@@ -1,187 +0,0 @@
-package org.apache.ignite.internal.processors.hadoop.fs;
-
-import java.io.Externalizable;
-import java.io.IOException;
-import java.io.ObjectInput;
-import java.io.ObjectOutput;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Collection;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.hadoop.fs.HadoopFileSystemFactory;
-import org.apache.ignite.internal.processors.hadoop.HadoopUtils;
-import org.apache.ignite.internal.processors.igfs.IgfsPaths;
-import org.apache.ignite.internal.processors.igfs.IgfsUtils;
-import org.apache.ignite.internal.util.typedef.F;
-import org.apache.ignite.internal.util.typedef.internal.A;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lifecycle.LifecycleAware;
-
-import static org.apache.ignite.internal.util.lang.GridFunc.nullifyEmpty;
-
-/**
- * The class is to be instantiated as a Spring beans, so it must have public zero-arg constructor.
- * The class is serializable as it will be transferred over the network as a part of {@link IgfsPaths} object.
- */
-public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<FileSystem>, Externalizable, LifecycleAware {
- /** Configuration of the secondary filesystem, never null. */
- protected final Configuration cfg = HadoopUtils.safeCreateConfiguration();
-
- /** */
- private URI uri;
-
- /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
- private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
- new HadoopLazyConcurrentMap.ValueFactory<String, FileSystem>() {
- @Override public FileSystem createValue(String key) {
- try {
- assert !F.isEmpty(key);
-
- return createFileSystem(key);
- }
- catch (IOException ioe) {
- throw new IgniteException(ioe);
- }
- }
- }
- );
-
- public DefaultHadoopFileSystemFactory() {
- //
- }
-
- @Override public FileSystem get(String userName) throws IOException {
- return fileSysLazyMap.getOrCreate(userName);
- }
-
- public void setUri(URI uri) {
- this.uri = uri;
- }
-
- /**
- * Convenience mathod, analog of {@link #setUri(URI)} with String type argument.
- * @param uriStr
- */
- public void setUri(String uriStr) {
- try {
- setUri(new URI(uriStr));
- }
- catch (URISyntaxException use) {
- throw new IgniteException(use);
- }
- }
-
- /**
- * Configuration(s) setter, to be invoked from Spring config.
- * @param cfgPaths
- */
- public void setCfgPaths(Collection<String> cfgPaths) {
- cfgPaths = nullifyEmpty(cfgPaths);
-
- if (cfgPaths == null)
- return;
-
- for (String confPath: cfgPaths) {
- confPath = nullifyEmpty(confPath);
-
- if (confPath != null) {
- URL url = U.resolveIgniteUrl(confPath);
-
- if (url == null) {
- // If secConfPath is given, it should be resolvable:
- throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
- "(ensure that it exists locally and you have read access to it): " + confPath);
- }
-
- cfg.addResource(url);
- }
- }
- }
-
- protected void init() throws IOException {
- String secUri = nullifyEmpty(uri == null ? null : uri.toString());
-
- A.ensure(cfg != null, "config");
-
- // if secondary fs URI is not given explicitly, try to get it from the configuration:
- if (secUri == null)
- uri = FileSystem.getDefaultUri(cfg);
- else {
- try {
- uri = new URI(secUri);
- }
- catch (URISyntaxException use) {
- throw new IOException("Failed to resolve secondary file system URI: " + secUri);
- }
- }
-
- // Disable caching:
- String prop = HadoopFileSystemsUtils.disableFsCachePropertyName(uri.getScheme());
-
- cfg.setBoolean(prop, true);
- }
-
- /**
- * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
- * @throws IOException
- */
- protected FileSystem createFileSystem(String userName) throws IOException {
- userName = IgfsUtils.fixUserName(nullifyEmpty(userName));
-
- final FileSystem fileSys;
-
- try {
- fileSys = FileSystem.get(uri, cfg, userName);
- }
- catch (InterruptedException e) {
- Thread.currentThread().interrupt();
-
- throw new IOException("Failed to create file system due to interrupt.", e);
- }
-
- return fileSys;
- }
-
- @Override public void writeExternal(ObjectOutput out) throws IOException {
- cfg.write(out);
-
- U.writeString(out, uri.toString());
- }
-
- @Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
- cfg.clear();
-
- cfg.readFields(in);
-
- String uriStr = U.readString(in);
-
- try {
- uri = new URI(uriStr);
- }
- catch (URISyntaxException use) {
- throw new IOException(use);
- }
- }
-
- @Override public void start() throws IgniteException {
- try {
- init();
- }
- catch (IOException ice) {
- throw new IgniteException(ice);
- }
- }
-
- @Override public void stop() throws IgniteException {
- try {
- fileSysLazyMap.close();
- }
- catch (IgniteCheckedException ice) {
- throw new IgniteException(ice);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
index 89eaf73..58b5120 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/HadoopLazyConcurrentMap.java
@@ -57,6 +57,10 @@ public class HadoopLazyConcurrentMap<K, V extends Closeable> {
assert getClass().getClassLoader() == Ignite.class.getClassLoader();
}
+ public int size () {
+ return map.size();
+ }
+
/**
* Gets cached or creates a new value of V.
* Never returns null.
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 7e5ef39..1ce0492 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -25,7 +25,6 @@ import java.io.IOException;
import java.io.OutputStream;
import java.lang.reflect.Field;
import java.net.URI;
-import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayDeque;
import java.util.Arrays;
@@ -63,7 +62,7 @@ import org.apache.ignite.configuration.FileSystemConfiguration;
import org.apache.ignite.configuration.IgniteConfiguration;
import org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem;
import org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem;
-import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
+import org.apache.ignite.hadoop.fs.v1.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEx;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsIpcIo;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutProc;
@@ -385,16 +384,16 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
if (mode != PRIMARY) {
DefaultHadoopFileSystemFactory fac = new DefaultHadoopFileSystemFactory();
+
fac.setUri(SECONDARY_URI);
- fac.setCfgPaths(Collections.singletonList(SECONDARY_CFG_PATH));
+ fac.setConfigPaths(Collections.singletonList(SECONDARY_CFG_PATH));
IgniteHadoopIgfsSecondaryFileSystem sec = new IgniteHadoopIgfsSecondaryFileSystem();
sec.setFsFactory(fac);
sec.setDfltUserName(SECONDARY_FS_USER);
- sec.start();
-
+ // NB: start() will be invoked upon IgfsImpl init.
cfg.setSecondaryFileSystem(sec);
}
@@ -412,7 +411,8 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
@Override public Object call() throws Exception {
return new IgniteHadoopFileSystem().getUri();
}
- }, IllegalStateException.class, "URI is null (was IgniteHadoopFileSystem properly initialized?).");
+ }, IllegalStateException.class,
+ "URI is null (was IgniteHadoopFileSystem properly initialized?) [closed=false]");
}
/** @throws Exception If failed. */
http://git-wip-us.apache.org/repos/asf/ignite/blob/89690022/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
index 20c2bd2..d8cf74c 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemShmemAbstractSelfTest.java
@@ -60,8 +60,6 @@ public abstract class IgniteHadoopFileSystemShmemAbstractSelfTest extends Ignite
*/
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
public void testOutOfResources() throws Exception {
- if (1 == 1) return;
-
final Collection<IpcEndpoint> eps = new LinkedList<>();
try {
[04/12] ignite git commit: 2206: intermediate commit for review.
Posted by vo...@apache.org.
2206: intermediate commit for review.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/d485969f
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/d485969f
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/d485969f
Branch: refs/heads/ignite-2206
Commit: d485969fbf8e95017e0601f1f0c35d31043599a7
Parents: 3b03b44
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 16:44:47 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 16:44:47 2015 +0300
----------------------------------------------------------------------
.../ignite/igfs/HadoopFileSystemFactory.java | 19 ++--
.../processors/igfs/IgfsHandshakeResponse.java | 19 +++-
.../internal/processors/igfs/IgfsImpl.java | 2 +-
.../internal/processors/igfs/IgfsPaths.java | 102 +++++++++----------
.../visor/node/VisorIgfsConfiguration.java | 38 +++----
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 31 +++++-
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 13 ++-
.../fs/DefaultHadoopFileSystemFactory.java | 1 -
8 files changed, 137 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
index 86d39e1..6720da9 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/HadoopFileSystemFactory.java
@@ -1,15 +1,14 @@
package org.apache.ignite.igfs;
-import java.io.Externalizable;
import java.io.IOException;
-import java.net.URI;
+import java.io.Serializable;
/**
- * This factory is {@link Externalizable} because it should be transferable over the network.
+ * This factory is {@link Serializable} because it should be transferable over the network.
*
* @param <T> The type
*/
-public interface HadoopFileSystemFactory <T> extends Externalizable {
+public interface HadoopFileSystemFactory <T> extends Serializable {
/**
* Gets the file system, possibly creating it or taking a cached instance.
* All the other data needed for the file system creation are expected to be contained
@@ -21,10 +20,10 @@ public interface HadoopFileSystemFactory <T> extends Externalizable {
*/
public T get(String userName) throws IOException;
- /**
- * Getter for the file system URI.
- *
- * @return The file system URI.
- */
- public URI uri();
+// /**
+// * Getter for the file system URI.
+// *
+// * @return The file system URI.
+// */
+// public URI uri();
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
index 1ba98ac..230f798 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsHandshakeResponse.java
@@ -36,6 +36,9 @@ public class IgfsHandshakeResponse implements Externalizable {
/** SECONDARY paths. */
private IgfsPaths paths;
+ /** */
+ private byte[] secondaryFileSystemFactoryBytes;
+
/** Server block size. */
private long blockSize;
@@ -55,10 +58,12 @@ public class IgfsHandshakeResponse implements Externalizable {
* @param paths Secondary paths.
* @param blockSize Server default block size.
*/
- public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, long blockSize, Boolean sampling) {
+ public IgfsHandshakeResponse(String igfsName, IgfsPaths paths, byte[] secondaryFileSystemFactoryBytes,
+ long blockSize, Boolean sampling) {
assert paths != null;
this.igfsName = igfsName;
+ this.secondaryFileSystemFactoryBytes = secondaryFileSystemFactoryBytes;
this.paths = paths;
this.blockSize = blockSize;
this.sampling = sampling;
@@ -92,12 +97,22 @@ public class IgfsHandshakeResponse implements Externalizable {
return sampling;
}
+ /**
+ *
+ * @return
+ */
+ public byte[] getSecondaryFileSystemFactoryBytes() {
+ return secondaryFileSystemFactoryBytes;
+ }
+
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeString(out, igfsName);
paths.writeExternal(out);
+ U.writeByteArray(out, secondaryFileSystemFactoryBytes);
+
out.writeLong(blockSize);
if (sampling != null) {
@@ -116,6 +131,8 @@ public class IgfsHandshakeResponse implements Externalizable {
paths.readExternal(in);
+ secondaryFileSystemFactoryBytes = U.readByteArray(in);
+
blockSize = in.readLong();
if (in.readBoolean())
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 7ea0333..15509ab 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -256,7 +256,7 @@ public final class IgfsImpl implements IgfsEx {
secondaryPaths = new IgfsPaths(
secondaryFs == null ? null : secondaryFs.properties(),
- secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
+ //secondaryFs == null ? null : secondaryFs.getSecondaryFileSystemFactory(),
dfltMode,
modeRslvr.modesOrdered());
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
index 809c7da..a9813d0 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsPaths.java
@@ -28,7 +28,7 @@ import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.ignite.igfs.HadoopFileSystemFactory;
+//import org.apache.ignite.igfs.HadoopFileSystemFactory;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.util.typedef.T2;
@@ -46,8 +46,8 @@ public class IgfsPaths <F> implements Externalizable {
@Deprecated
private Map<String, String> props;
- /** */
- private HadoopFileSystemFactory<F> factory;
+// /** */
+// private HadoopFileSystemFactory<F> factory;
/** Default IGFS mode. */
private IgfsMode dfltMode;
@@ -70,11 +70,11 @@ public class IgfsPaths <F> implements Externalizable {
* @param pathModes Path modes.
*/
public IgfsPaths(Map<String, String> props,
- HadoopFileSystemFactory<F> factory,
+ //HadoopFileSystemFactory<F> factory,
IgfsMode dfltMode,
@Nullable List<T2<IgfsPath, IgfsMode>> pathModes) {
this.props = props;
- this.factory = factory;
+ //this.factory = factory;
this.dfltMode = dfltMode;
this.pathModes = pathModes;
}
@@ -103,20 +103,20 @@ public class IgfsPaths <F> implements Externalizable {
return pathModes;
}
- /**
- * Getter for factory.
- *
- * @return The factory.
- */
- public HadoopFileSystemFactory<F> factory() {
- return factory;
- }
+// /**
+// * Getter for factory.
+// *
+// * @return The factory.
+// */
+// public HadoopFileSystemFactory<F> factory() {
+// return factory;
+// }
/** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeStringMap(out, props);
- writeFactory(out);
+// writeFactory(out);
U.writeEnum(out, dfltMode);
@@ -133,30 +133,30 @@ public class IgfsPaths <F> implements Externalizable {
out.writeBoolean(false);
}
- /**
- *
- * @param out
- * @throws IOException
- */
- private void writeFactory(ObjectOutput out) throws IOException {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- ObjectOutput oo = new ObjectOutputStream(baos);
- try {
- oo.writeObject(factory);
- }
- finally {
- oo.close();
- }
-
- U.writeByteArray(out, baos.toByteArray());
- }
+// /**
+// *
+// * @param out
+// * @throws IOException
+// */
+// private void writeFactory(ObjectOutput out) throws IOException {
+// ByteArrayOutputStream baos = new ByteArrayOutputStream();
+//
+// ObjectOutput oo = new ObjectOutputStream(baos);
+// try {
+// oo.writeObject(factory);
+// }
+// finally {
+// oo.close();
+// }
+//
+// U.writeByteArray(out, baos.toByteArray());
+// }
/** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
props = U.readStringMap(in);
- readFactory(in);
+// readFactory(in);
dfltMode = IgfsMode.fromOrdinal(in.readByte());
@@ -176,22 +176,22 @@ public class IgfsPaths <F> implements Externalizable {
}
}
- /**
- *
- * @param in
- * @throws IOException
- * @throws ClassNotFoundException
- */
- private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
- byte[] factoryBytes = U.readByteArray(in);
-
- ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
-
- try {
- factory = (HadoopFileSystemFactory<F>) oi.readObject();
- }
- finally {
- oi.close();
- }
- }
+// /**
+// *
+// * @param in
+// * @throws IOException
+// * @throws ClassNotFoundException
+// */
+// private void readFactory(ObjectInput in) throws IOException, ClassNotFoundException {
+// byte[] factoryBytes = U.readByteArray(in);
+//
+// ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+//
+// try {
+// factory = (HadoopFileSystemFactory<F>) oi.readObject();
+// }
+// finally {
+// oi.close();
+// }
+// }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 4a2e7b1..08792a4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -75,7 +75,7 @@ public class VisorIgfsConfiguration implements Serializable {
// /** User name for the secondary hadoop file system config. */
// private String secondaryHadoopFileSysUserName;
- private HadoopFileSystemFactory factory;
+// private HadoopFileSystemFactory factory;
/** IGFS instance mode. */
private IgfsMode dfltMode;
@@ -146,16 +146,16 @@ public class VisorIgfsConfiguration implements Serializable {
IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
- if (secFs != null) {
- //Map<String, String> props = secFs.properties();
-
- //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
- //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
- //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
-
- // Just take and save the factory object:
- cfg.factory = secFs.getSecondaryFileSystemFactory();
- }
+// if (secFs != null) {
+// //Map<String, String> props = secFs.properties();
+//
+// //cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
+// //cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
+// //cfg.secondaryHadoopFileSysUserName = props.get(SECONDARY_FS_USER_NAME);
+//
+// // Just take and save the factory object:
+// cfg.factory = secFs.getSecondaryFileSystemFactory();
+// }
cfg.dfltMode = igfs.getDefaultMode();
cfg.pathModes = igfs.getPathModes();
@@ -277,14 +277,14 @@ public class VisorIgfsConfiguration implements Serializable {
// return secondaryHadoopFileSysCfgPath;
// }
- /**
- *
- * @param <T>
- * @return
- */
- public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
- return factory;
- }
+// /**
+// *
+// * @param <T>
+// * @return
+// */
+// public <T> HadoopFileSystemFactory<T> secondaryFileSystemFactory() {
+// return factory;
+// }
/**
* @return IGFS instance mode.
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 2fcf774..20e2011 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -18,9 +18,12 @@
package org.apache.ignite.hadoop.fs.v1;
import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
import java.io.OutputStream;
import java.net.URI;
import java.net.URISyntaxException;
@@ -331,7 +334,11 @@ public class IgniteHadoopFileSystem extends FileSystem {
// String secUri = props.get(SECONDARY_FS_URI);
// String secConfPath = props.get(SECONDARY_FS_CONFIG_PATH);
- HadoopFileSystemFactory<FileSystem> factory = paths.factory();
+ byte[] secFsFacoryBytes = handshake.getSecondaryFileSystemFactoryBytes();
+
+
+
+ //HadoopFileSystemFactory<FileSystem> factory = paths.factory();
A.ensure(factory != null, "Secondary file system factory should not be null.");
@@ -343,6 +350,10 @@ public class IgniteHadoopFileSystem extends FileSystem {
//SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
secondaryFs = factory.get(user); //secProvider.createFileSystem(user);
+
+ URI uri2 = secondaryFs.getUri();
+
+ assert secondaryUri.equals(uri2);
}
catch (IOException e) {
if (!mgmt)
@@ -361,6 +372,24 @@ public class IgniteHadoopFileSystem extends FileSystem {
}
}
+ /**
+ *
+ * @param in
+ * @throws IOException
+ * @throws ClassNotFoundException
+ */
+ static HadoopFileSystemFactory readFactory(byte[] factoryBytes) throws IOException, ClassNotFoundException {
+ ObjectInput oi = new ObjectInputStream(new ByteArrayInputStream(factoryBytes));
+
+ try {
+ return (HadoopFileSystemFactory<F>) oi.readObject();
+ }
+ finally {
+ oi.close();
+ }
+ }
+
+
/** {@inheritDoc} */
@Override protected void checkPath(Path path) {
URI uri = path.toUri();
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index 865a2bc..ae97464 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -58,6 +58,7 @@ import org.apache.ignite.igfs.IgfsFile;
import org.apache.ignite.igfs.IgfsMode;
import org.apache.ignite.igfs.IgfsPath;
import org.apache.ignite.internal.igfs.common.IgfsLogger;
+import org.apache.ignite.internal.processors.hadoop.fs.DefaultHadoopFileSystemFactory;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsEndpoint;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsInputStream;
import org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsOutputStream;
@@ -345,17 +346,21 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
A.ensure(secondaryUri != null, "File system factory uri should not be null.");
- secondaryUri = factory.uri();
-
- A.ensure(secondaryUri != null, "Secondary file system uri should not be null.");
+ //secondaryUri = factory.uri();
try {
//SecondaryFileSystemProvider secProvider = new SecondaryFileSystemProvider(secUri, secConfPath);
secondaryFs = factory.get(user);
- //secondaryFs = secProvider.createAbstractFileSystem(user);
+ secondaryUri = secondaryFs.getUri();
+
+ assert secondaryUri != null;
+ URI uri2 = ((DefaultHadoopFileSystemFactory)factory).uri();
+ assert secondaryUri.equals(uri2);
+
+ //secondaryFs = secProvider.createAbstractFileSystem(user);
//secondaryUri = secProvider.uri();
}
catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/ignite/blob/d485969f/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
index 246637d..0b93cba 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/fs/DefaultHadoopFileSystemFactory.java
@@ -106,7 +106,6 @@ public class DefaultHadoopFileSystemFactory implements HadoopFileSystemFactory<F
}
}
-
protected void init() throws IOException {
String secUri = nullifyEmpty(uri == null ? null : uri.toString());
[03/12] ignite git commit: Merge branch 'ignite-1.5' of
https://github.com/apache/ignite into ignite-2206
Posted by vo...@apache.org.
Merge branch 'ignite-1.5' of https://github.com/apache/ignite into ignite-2206
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/3b03b449
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/3b03b449
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/3b03b449
Branch: refs/heads/ignite-2206
Commit: 3b03b44988e0d41f6c2d3d52ecf7f4d888646600
Parents: 31c40d6 6ad758c
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Dec 22 16:06:20 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Dec 22 16:06:20 2015 +0300
----------------------------------------------------------------------
.../configuration/CacheConfiguration.java | 44 ++-
.../internal/binary/BinaryClassDescriptor.java | 56 +++-
.../internal/binary/BinaryObjectImpl.java | 20 +-
.../ignite/internal/binary/BinaryUtils.java | 11 +
.../internal/binary/GridBinaryMarshaller.java | 10 +
.../processors/cache/IgniteCacheProxy.java | 8 +-
.../binary/CacheObjectBinaryProcessorImpl.java | 3 +
.../cache/query/GridCacheQueryManager.java | 8 +-
.../processors/query/GridQueryProcessor.java | 10 +-
.../ignite/internal/util/IgniteUtils.java | 6 +-
.../cache/VisorCacheQueryConfiguration.java | 11 +
.../cache/VisorCacheStoreConfiguration.java | 13 +-
.../internal/visor/query/VisorQueryJob.java | 6 +
.../binary/BinaryMarshallerSelfTest.java | 110 +++++++
.../IgniteCacheBinaryObjectsScanSelfTest.java | 137 +++++++++
.../processors/query/h2/IgniteH2Indexing.java | 118 ++++++--
.../processors/query/h2/sql/GridSqlQuery.java | 3 +
.../query/h2/sql/GridSqlQuerySplitter.java | 113 ++++---
.../cache/IgniteCacheAbstractQuerySelfTest.java | 300 +++++++++++++------
...teCacheFullTextQueryNodeJoiningSelfTest.java | 145 +++++++++
.../IgniteCacheReplicatedQuerySelfTest.java | 10 +-
.../local/IgniteCacheLocalQuerySelfTest.java | 2 +-
.../query/IgniteSqlSchemaIndexingTest.java | 240 +++++++++++++++
.../query/IgniteSqlSplitterSelfTest.java | 52 ++++
.../query/h2/sql/GridQueryParsingTest.java | 9 +-
.../IgniteBinaryCacheQueryTestSuite.java | 5 +-
.../IgniteCacheQuerySelfTestSuite.java | 2 +
modules/platforms/cpp/common/configure.ac | 2 +-
modules/platforms/cpp/core-test/configure.ac | 2 +-
modules/platforms/cpp/core/configure.ac | 2 +-
modules/platforms/cpp/examples/configure.ac | 2 +-
modules/platforms/cpp/ignite/configure.ac | 2 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Apache.Ignite/Properties/AssemblyInfo.cs | 6 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../Properties/AssemblyInfo.cs | 6 +-
.../commands/cache/VisorCacheCommand.scala | 14 +-
.../config/benchmark-multicast.properties | 6 +-
pom.xml | 2 +-
42 files changed, 1275 insertions(+), 251 deletions(-)
----------------------------------------------------------------------
[12/12] ignite git commit: IGNITE-2206: WIP.
Posted by vo...@apache.org.
IGNITE-2206: WIP.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/31d3289d
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/31d3289d
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/31d3289d
Branch: refs/heads/ignite-2206
Commit: 31d3289df789eab0ba7bbe8350507c69cc0ba845
Parents: 13f8170
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Dec 23 13:40:50 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Dec 23 13:40:50 2015 +0300
----------------------------------------------------------------------
.../fs/CachingHadoopFileSystemFactory.java | 10 +-
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 227 ++++++-------------
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 1 +
.../hadoop/IgfsSecondaryFileSystemEx.java | 15 --
4 files changed, 78 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ignite/blob/31d3289d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
index 3bad850..1e97b30 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/CachingHadoopFileSystemFactory.java
@@ -139,18 +139,21 @@ public class CachingHadoopFileSystemFactory implements HadoopFileSystemFactory,
return fileSys;
}
+ /** {@inheritDoc} */
@Override public void writeExternal(ObjectOutput out) throws IOException {
U.writeString(out, uriStr);
U.writeCollection(out, cfgPathStr);
}
+ /** {@inheritDoc} */
@Override public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
uriStr = U.readString(in);
cfgPathStr = new ArrayList(U.readCollection(in));
}
+ /** {@inheritDoc} */
@Override public void start() throws IgniteException {
cfg = HadoopUtils.safeCreateConfiguration();
@@ -161,13 +164,15 @@ public class CachingHadoopFileSystemFactory implements HadoopFileSystemFactory,
if (url == null) {
// If secConfPath is given, it should be resolvable:
- throw new IllegalArgumentException("Failed to resolve secondary file system configuration path " +
-
+ throw new IgniteException("Failed to resolve secondary file system configuration path " +
"(ensure that it exists locally and you have read access to it): " + confPath);
}
cfg.addResource(url);
}
+ else {
+ // TODO: Throw exception.
+ }
}
}
@@ -191,6 +196,7 @@ public class CachingHadoopFileSystemFactory implements HadoopFileSystemFactory,
cfg.setBoolean(prop, true);
}
+ /** {@inheritDoc} */
@Override public void stop() throws IgniteException {
try {
fileSysLazyMap.close();
http://git-wip-us.apache.org/repos/asf/ignite/blob/31d3289d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 756989c..aa1952d 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -17,15 +17,6 @@
package org.apache.ignite.hadoop.fs;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.ParentNotDirectoryException;
@@ -52,11 +43,19 @@ import org.apache.ignite.internal.processors.igfs.IgfsFileImpl;
import org.apache.ignite.internal.processors.igfs.IgfsFileInfo;
import org.apache.ignite.internal.processors.igfs.IgfsUtils;
import org.apache.ignite.internal.util.typedef.F;
-
import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.lifecycle.LifecycleAware;
import org.jetbrains.annotations.Nullable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_GROUP_NAME;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_PERMISSION;
import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
@@ -66,54 +65,17 @@ import static org.apache.ignite.internal.processors.igfs.IgfsEx.PROP_USER_NAME;
* In fact, this class deals with different FileSystems depending on the user context,
* see {@link IgfsUserContext#currentUser()}.
*/
-public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem,
- LifecycleAware, HadoopPayloadAware {
-// /** Properties of file system, see {@link #properties()}
-// * */
-// private final Map<String, String> props = new HashMap<>();
-
- /** Secondary file system provider. */
- //private SecondaryFileSystemProvider secProvider;
-
- /** FileSystem instance created for the default user.
- * Stored outside the fileSysLazyMap due to performance reasons. */
- private FileSystem dfltFs;
-
-// /** */
-// private String uriStr;
-//
-// /** Note: */
-// private URI uri;
-//
-// /** */
-// private Collection<String> cfgPathsStr;
-//
-// /** */
-// private @Nullable Collection<URI> cfgPaths;
-
+public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, LifecycleAware,
+ HadoopPayloadAware {
/** The default user name. It is used if no user context is set. */
- private String usrName = IgfsUtils.fixUserName(null);
+ private String dfltUsrName = IgfsUtils.fixUserName(null);
/** */
private HadoopFileSystemFactory fsFactory;
- private final AtomicBoolean started = new AtomicBoolean();
-
-// /** Lazy per-user cache for the file systems. It is cleared and nulled in #close() method. */
-// private final HadoopLazyConcurrentMap<String, FileSystem> fileSysLazyMap = new HadoopLazyConcurrentMap<>(
-// new ValueFactory<String, FileSystem>() {
-// @Override public FileSystem createValue(String key) {
-// try {
-// assert !F.isEmpty(key);
-//
-// return secProvider.createFileSystem(key);
-// }
-// catch (IOException ioe) {
-// throw new IgniteException(ioe);
-// }
-// }
-// }
-// );
+ /** FileSystem instance created for the default user. Stored outside due to performance reasons. */
+ // TODO: Remove.
+ private volatile FileSystem dfltFs;
/**
* Default constructor for Spring.
@@ -139,6 +101,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
* @param cfgPath Additional path to Hadoop configuration.
* @throws IgniteCheckedException In case of error.
*/
+ @Deprecated
public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath)
throws IgniteCheckedException {
this(uri, cfgPath, null);
@@ -168,6 +131,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
setUserName(userName);
}
+ // TODO: Add getter.
+ // TODO: Add docs.
/**
*
* @param factory
@@ -178,33 +143,19 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
this.fsFactory = factory;
}
+ // TODO: Add getter.
+ // TODO: Add docs.
+ // TODO: Rename to "setDefaultUserName"
+
/**
*
* @param usrName
*/
public void setUserName(String usrName) {
// TODO: Move fix to start routine.
- this.usrName = IgfsUtils.fixUserName(usrName);
+ this.dfltUsrName = IgfsUtils.fixUserName(usrName);
}
-// /**
-// * Sets the file system properties.
-// */
-// private void setProperties() {
-// String uri = this.uri.toString();
-//
-// if (!uri.endsWith("/"))
-// uri += "/";
-//
-// String cfgPath = secProvider.configurationPath();
-//
-// if (cfgPath != null)
-// props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
-//
-// props.put(SECONDARY_FS_URI, uri);
-// props.put(SECONDARY_FS_USER_NAME, dfltUserName);
-// }
-
/**
* Convert IGFS path into Hadoop path.
*
@@ -212,7 +163,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
* @return Hadoop path.
*/
private Path convert(IgfsPath path) {
- URI uri = fileSysForUser().getUri();
+ URI uri = fileSystemForUser().getUri();
return new Path(uri.getScheme(), uri.getAuthority(), path.toString());
}
@@ -266,7 +217,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public boolean exists(IgfsPath path) {
try {
- return fileSysForUser().exists(convert(path));
+ return fileSystemForUser().exists(convert(path));
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]");
@@ -277,7 +228,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
@Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
HadoopIgfsProperties props0 = new HadoopIgfsProperties(props);
- final FileSystem fileSys = fileSysForUser();
+ final FileSystem fileSys = fileSystemForUser();
try {
if (props0.userName() != null || props0.groupName() != null)
@@ -298,7 +249,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
@Override public void rename(IgfsPath src, IgfsPath dest) {
// Delegate to the secondary file system.
try {
- if (!fileSysForUser().rename(convert(src), convert(dest)))
+ if (!fileSystemForUser().rename(convert(src), convert(dest)))
throw new IgfsException("Failed to rename (secondary file system returned false) " +
"[src=" + src + ", dest=" + dest + ']');
}
@@ -310,7 +261,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public boolean delete(IgfsPath path, boolean recursive) {
try {
- return fileSysForUser().delete(convert(path), recursive);
+ return fileSystemForUser().delete(convert(path), recursive);
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]");
@@ -320,7 +271,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public void mkdirs(IgfsPath path) {
try {
- if (!fileSysForUser().mkdirs(convert(path)))
+ if (!fileSystemForUser().mkdirs(convert(path)))
throw new IgniteException("Failed to make directories [path=" + path + "]");
}
catch (IOException e) {
@@ -331,7 +282,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
try {
- if (!fileSysForUser().mkdirs(convert(path), new HadoopIgfsProperties(props).permission()))
+ if (!fileSystemForUser().mkdirs(convert(path), new HadoopIgfsProperties(props).permission()))
throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]");
}
catch (IOException e) {
@@ -342,7 +293,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public Collection<IgfsPath> listPaths(IgfsPath path) {
try {
- FileStatus[] statuses = fileSysForUser().listStatus(convert(path));
+ FileStatus[] statuses = fileSystemForUser().listStatus(convert(path));
if (statuses == null)
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
@@ -365,7 +316,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public Collection<IgfsFile> listFiles(IgfsPath path) {
try {
- FileStatus[] statuses = fileSysForUser().listStatus(convert(path));
+ FileStatus[] statuses = fileSystemForUser().listStatus(convert(path));
if (statuses == null)
throw new IgfsPathNotFoundException("Failed to list files (path not found): " + path);
@@ -392,13 +343,13 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) {
- return new HadoopIgfsSecondaryFileSystemPositionedReadable(fileSysForUser(), convert(path), bufSize);
+ return new HadoopIgfsSecondaryFileSystemPositionedReadable(fileSystemForUser(), convert(path), bufSize);
}
/** {@inheritDoc} */
@Override public OutputStream create(IgfsPath path, boolean overwrite) {
try {
- return fileSysForUser().create(convert(path), overwrite);
+ return fileSystemForUser().create(convert(path), overwrite);
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]");
@@ -412,8 +363,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
new HadoopIgfsProperties(props != null ? props : Collections.<String, String>emptyMap());
try {
- return fileSysForUser().create(convert(path), props0.permission(), overwrite, bufSize,
- (short)replication, blockSize, null);
+ return fileSystemForUser().create(convert(path), props0.permission(), overwrite, bufSize,
+ (short) replication, blockSize, null);
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props +
@@ -426,7 +377,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
@Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
@Nullable Map<String, String> props) {
try {
- return fileSysForUser().append(convert(path), bufSize);
+ return fileSystemForUser().append(convert(path), bufSize);
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
@@ -436,7 +387,7 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
/** {@inheritDoc} */
@Override public IgfsFile info(final IgfsPath path) {
try {
- final FileStatus status = fileSysForUser().getFileStatus(convert(path));
+ final FileStatus status = fileSystemForUser().getFileStatus(convert(path));
if (status == null)
return null;
@@ -511,71 +462,37 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
try {
// We don't use FileSystem#getUsed() since it counts only the files
// in the filesystem root, not all the files recursively.
- return fileSysForUser().getContentSummary(new Path("/")).getSpaceConsumed();
+ return fileSystemForUser().getContentSummary(new Path("/")).getSpaceConsumed();
}
catch (IOException e) {
throw handleSecondaryFsError(e, "Failed to get used space size of file system.");
}
}
-// /** {@inheritDoc} */
-// @Override public Map<String, String> properties() {
-// return Collections.emptyMap();
-// }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteException {
- Exception e = null;
-
- try {
- if (dfltFs != null)
- dfltFs.close();
- }
- catch (Exception e0) {
- e = e0;
- }
-
- try {
- if (fsFactory instanceof LifecycleAware)
- ((LifecycleAware)fsFactory).stop();
- }
- catch (IgniteException ie) {
- if (e == null)
- e = ie;
- }
-
- if (e != null)
- throw new IgniteException(e);
- }
-
/**
* Gets the underlying {@link FileSystem}.
* This method is used solely for testing.
* @return the underlying Hadoop {@link FileSystem}.
*/
public FileSystem fileSystem() {
- return fileSysForUser();
+ return fileSystemForUser();
}
/**
* Gets the FileSystem for the current context user.
* @return the FileSystem instance, never null.
*/
- private FileSystem fileSysForUser() {
- assert started.get(); // Ensure the Fs is started.
-
+ private FileSystem fileSystemForUser() {
String user = IgfsUserContext.currentUser();
if (F.isEmpty(user))
- user = usrName; // default is never empty.
+ user = dfltUsrName; // default is never empty.
assert !F.isEmpty(user);
- if (F.eq(user, usrName))
+ if (F.eq(user, dfltUsrName))
return dfltFs; // optimization
- //assert fsFactory.uri() != null : "uri!";
-
try {
return fsFactory.create(user);
}
@@ -584,48 +501,42 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
}
}
- /**
- * Should be invoked by client (from Spring?) after all the setters invoked.
- *
- * @throws IgniteCheckedException
- */
- @Override public void start() {
+ /** {@inheritDoc} */
+ @Override public void start() throws IgniteException {
// #start() should not ever be invoked if these properties are not set:
A.ensure(fsFactory != null, "factory");
- A.ensure(usrName != null, "dfltUserName");
-
- // Avoid
- if (started.compareAndSet(false, true)) {
- if (fsFactory instanceof LifecycleAware)
- ((LifecycleAware) fsFactory).start();
-
- try {
- //this.secProvider = new SecondaryFileSystemProvider(uri, cfgPath);
-
- // File system creation for the default user name.
- // The value is *not* stored in the 'fileSysLazyMap' cache, but saved in field:
- //this.dfltFs = secProvider.createFileSystem(dfltUserName);
- this.dfltFs = fsFactory.create(usrName);
+ A.ensure(dfltUsrName != null, "userName");
- assert dfltFs != null;
- }
- catch (IOException e) {
- throw new IgniteException(e);
- }
- }
+ if (fsFactory instanceof LifecycleAware)
+ ((LifecycleAware) fsFactory).start();
}
-// /** {@inheritDoc} */
-// @Nullable @Override public HadoopFileSystemFactory<FileSystem> getSecondaryFileSystemFactory() {
-// return fsFactory;
-// }
+ /** {@inheritDoc} */
+ @Override public void stop() throws IgniteException {
+ Exception e = null;
+ try {
+ if (dfltFs != null)
+ dfltFs.close();
+ }
+ catch (Exception e0) {
+ e = e0;
+ }
+ try {
+ if (fsFactory instanceof LifecycleAware)
+ ((LifecycleAware)fsFactory).stop();
+ }
+ catch (IgniteException ie) {
+ if (e == null)
+ e = ie;
+ }
- @Override public void stop() throws IgniteException {
- close();
+ if (e != null)
+ throw new IgniteException(e);
}
+ /** {@inheritDoc} */
@Override public HadoopFileSystemFactory getPayload() {
return fsFactory;
}
http://git-wip-us.apache.org/repos/asf/ignite/blob/31d3289d/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 9463412..1546995 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -164,6 +164,7 @@ public class IgniteHadoopFileSystem extends FileSystem {
/** IGFS mode resolver. */
private IgfsModeResolver modeRslvr;
+ // TODO: Secondary file system must be changed to factory.
/** Secondary file system instance. */
private FileSystem secondaryFs;
http://git-wip-us.apache.org/repos/asf/ignite/blob/31d3289d/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
deleted file mode 100644
index 582e798..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/IgfsSecondaryFileSystemEx.java
+++ /dev/null
@@ -1,15 +0,0 @@
-//package org.apache.ignite.internal.processors.hadoop;
-//
-//import org.apache.ignite.IgniteCheckedException;
-//import org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem;
-//
-///**
-// *
-// */
-//public interface IgfsSecondaryFileSystemEx extends IgfsSecondaryFileSystem {
-// /**
-// *
-// * @throws IgniteCheckedException
-// */
-// public void start() throws IgniteCheckedException;
-//}