You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/03/02 19:22:36 UTC
[1/2] incubator-ignite git commit: [IGNITE-349]: secondary filesystem
configuration made optional. The logic of the filesystem construction made
common,
and is now reused in all places where used. Constants also now specified in
single place and reused.
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-349 [created] e58cbfb4c
[IGNITE-349]: secondary filesystem configuration made optional. The logic of the filesystem construction made common, and is now reused in all places where used. Constants also now specified in single place and reused.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/51985d5b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/51985d5b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/51985d5b
Branch: refs/heads/ignite-349
Commit: 51985d5b44234187dac0b4fb215297d42ed1e822
Parents: b2675bc
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Mar 2 21:20:03 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Mar 2 21:20:03 2015 +0300
----------------------------------------------------------------------
modules/hadoop/pom.xml | 15 +-
.../hadoop/IgfsHadoopFileSystemWrapper.java | 38 ++---
.../hadoop/SecondaryFileSystemProvider.java | 152 +++++++++++++++++++
.../igfs/hadoop/v1/IgfsHadoopFileSystem.java | 36 +----
.../igfs/hadoop/v2/IgfsHadoopFileSystem.java | 44 +-----
5 files changed, 189 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/51985d5b/modules/hadoop/pom.xml
----------------------------------------------------------------------
diff --git a/modules/hadoop/pom.xml b/modules/hadoop/pom.xml
index 04ab706..22e84ca 100644
--- a/modules/hadoop/pom.xml
+++ b/modules/hadoop/pom.xml
@@ -44,6 +44,14 @@
<dependency>
<groupId>org.apache.ignite</groupId>
+ <artifactId>ignite-core</artifactId>
+ <version>${ignite.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.ignite</groupId>
<artifactId>ignite-log4j</artifactId>
<version>${ignite.version}</version>
</dependency>
@@ -109,13 +117,6 @@
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.apache.ignite</groupId>
- <artifactId>ignite-core</artifactId>
- <version>${ignite.version}</version>
- <type>test-jar</type>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/51985d5b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
index 29dfde5..2577516 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
@@ -17,7 +17,6 @@
package org.apache.ignite.igfs.hadoop;
-import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.*;
@@ -27,7 +26,6 @@ import org.apache.ignite.igfs.*;
import org.apache.ignite.internal.igfs.hadoop.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
import java.io.*;
@@ -38,11 +36,6 @@ import java.util.*;
* Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as {@link org.apache.ignite.igfs.Igfs}.
*/
public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable {
- /** Property name for path to Hadoop configuration. */
- public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
-
- /** Property name for URI of file system. */
- public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
/** Hadoop file system. */
private final FileSystem fileSys;
@@ -51,6 +44,16 @@ public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable {
private final Map<String, String> props = new HashMap<>();
/**
+ * Simple constructor that is to be used by default.
+ *
+ * @param uri URI of file system.
+ * @throws IgniteCheckedException In case of error.
+ */
+ public IgfsHadoopFileSystemWrapper(@NotNull String uri) throws IgniteCheckedException {
+ this(uri, null);
+ }
+
+ /**
* Constructor.
*
* @param uri URI of file system.
@@ -58,25 +61,16 @@ public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable {
* @throws IgniteCheckedException In case of error.
*/
public IgfsHadoopFileSystemWrapper(@Nullable String uri, @Nullable String cfgPath) throws IgniteCheckedException {
- Configuration cfg = new Configuration();
-
- if (cfgPath != null)
- cfg.addResource(U.resolveIgniteUrl(cfgPath));
-
try {
- fileSys = uri == null ? FileSystem.get(cfg) : FileSystem.get(new URI(uri), cfg);
+ SecondaryFileSystemProvider secondaryFileSystemProvider =
+ new SecondaryFileSystemProvider(uri, cfgPath, false); // TODO: may be disableCaching == true there?
+
+ fileSys = secondaryFileSystemProvider.getFileSystem();
+ secondaryFileSystemProvider.exportProperties(props);
}
- catch (IOException | URISyntaxException e) {
+ catch (IOException e) {
throw new IgniteCheckedException(e);
}
-
- uri = fileSys.getUri().toString();
-
- if (!uri.endsWith("/"))
- uri += "/";
-
- props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
- props.put(SECONDARY_FS_URI, uri);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/51985d5b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
new file mode 100644
index 0000000..81a12ea
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.hadoop;
+
+import com.sun.istack.NotNull;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.igfs.IgfsConst;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.Map;
+
+/**
+ * Encapsulates logic of secondary filesystem creation.
+ */
+public class SecondaryFileSystemProvider {
+
+ private @NotNull final Configuration cfg = new Configuration();
+ private @NotNull final URI secondaryFsUri;
+ private @Nullable final String secConfPath;
+ private final boolean disableCaching;
+
+ /**
+ * Creates new provider with given config parameters.
+ * The configuration URL is optional. The filesystem URI must be specified either explicitly or in the
+ * configuration provided.
+ *
+ * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified
+ * as "fs.defaultFS" property in the provided configuration.
+ * @param secConfPath the secondary Fs config URL (optional)
+ * @param disableCaching 'true' to disable caching in the created Hadoop filesystem. Used to avoid double caching.
+ * @throws IOException
+ */
+ public SecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath,
+ boolean disableCaching) throws IOException {
+ if (secUri == null && secConfPath == null) {
+ throw new IllegalArgumentException("Failed to get secondary file system: neither " +
+ "secondaryUri nor secondaryConfigPath given.");
+ }
+
+ this.disableCaching = disableCaching;
+
+ if (secConfPath != null) {
+ URL url = U.resolveIgniteUrl(secConfPath);
+ if (url == null) {
+ // If secConfPath is given, it should be resolvable:
+ throw new IllegalArgumentException("Failed to resolve secondary Fs " +
+ "configuration path [" + secConfPath + "]");
+ }
+ cfg.addResource(url);
+ }
+
+ this.secConfPath = secConfPath;
+
+ try {
+ // if secondary fs URI is not given explicitly, try to take it from the configuration:
+ secondaryFsUri = (secUri == null) ? FileSystem.getDefaultUri(cfg) : new URI(secUri);
+ } catch (URISyntaxException use) {
+ throw new IOException("Failed to resolve secondary file system URI: " + secUri);
+ }
+
+ if (secondaryFsUri == null) {
+ throw new IllegalArgumentException("Failed to get secondary file system URI: it is neither given " +
+ "explicitly nor specified in the configuration ["+secConfPath+"]");
+ }
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ public @NotNull FileSystem getFileSystem() throws IOException {
+ if (disableCaching) {
+ disableCaching();
+ }
+
+ FileSystem fileSys = FileSystem.get(secondaryFsUri, cfg);
+
+ return fileSys;
+ }
+
+ /**
+ * Exports properties of the filesystem to the given map.
+ * @param props
+ */
+ public void exportProperties(@NotNull Map<String,String> props) {
+ props.put(IgfsConst.SECONDARY_FS_CONFIG_PATH, secConfPath);
+ props.put(IgfsConst.SECONDARY_FS_URI, getSecondaryFsUriStr());
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ public @NotNull AbstractFileSystem getAbstractFileSystem() throws IOException {
+ if (disableCaching) {
+ disableCaching();
+ }
+
+ AbstractFileSystem secondaryFs = AbstractFileSystem.get(secondaryFsUri, cfg);
+
+ return secondaryFs;
+ }
+
+ private void disableCaching() {
+ String prop = String.format("fs.%s.impl.disable.cache", secondaryFsUri.getScheme());
+
+ cfg.setBoolean(prop, true);
+ }
+
+ /**
+ * @return the secondary fs URI, never null.
+ */
+ public @NotNull URI getSecondaryFsUri() {
+ return secondaryFsUri;
+ }
+
+ /**
+ *
+ * @return "slashified" secondary Fs URI in String form, never null.
+ */
+ private @NotNull String getSecondaryFsUriStr() {
+ String uri = secondaryFsUri.toString();
+
+ if (!uri.endsWith("/"))
+ uri += "/";
+
+ return uri;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/51985d5b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
index 1648bdc..cdfe1a1 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
@@ -282,39 +282,15 @@ public class IgfsHadoopFileSystem extends FileSystem {
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
- String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);
-
- if (secConfPath == null)
- throw new IOException("Failed to connect to the secondary file system because configuration " +
- "path is not provided.");
-
- if (secUri == null)
- throw new IOException("Failed to connect to the secondary file system because URI is not " +
- "provided.");
+ String secUri = props.get(IgfsConst.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
try {
- secondaryUri = new URI(secUri);
-
- URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath);
-
- Configuration conf = new Configuration();
-
- if (secondaryCfgUrl != null)
- conf.addResource(secondaryCfgUrl);
-
- String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());
+ SecondaryFileSystemProvider secondaryFileSystemProvider =
+ new SecondaryFileSystemProvider(secUri, secConfPath, true);
- conf.setBoolean(prop, true);
-
- secondaryFs = FileSystem.get(secondaryUri, conf);
- }
- catch (URISyntaxException ignore) {
- if (!mgmt)
- throw new IOException("Failed to resolve secondary file system URI: " + secUri);
- else
- LOG.warn("Visor failed to create secondary file system (operations on paths with PROXY mode " +
- "will have no effect).");
+ secondaryFs = secondaryFileSystemProvider.getFileSystem();
+ secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri();
}
catch (IOException e) {
if (!mgmt)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/51985d5b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
index 5475cf4..691705b 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
@@ -270,52 +270,22 @@ public class IgfsHadoopFileSystem extends AbstractFileSystem implements Closeabl
for (T2<IgfsPath, IgfsMode> pathMode : paths.pathModes()) {
IgfsMode mode = pathMode.getValue();
- initSecondary |= mode == PROXY;
+ initSecondary |= (mode == PROXY);
}
}
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_URI);
- String secConfPath = props.get(IgfsHadoopFileSystemWrapper.SECONDARY_FS_CONFIG_PATH);
-
- if (secConfPath == null)
- throw new IOException("Failed to connect to the secondary file system because configuration " +
- "path is not provided.");
-
- if (secUri == null)
- throw new IOException("Failed to connect to the secondary file system because URI is not " +
- "provided.");
-
- if (secConfPath == null)
- throw new IOException("Failed to connect to the secondary file system because configuration " +
- "path is not provided.");
-
- if (secUri == null)
- throw new IOException("Failed to connect to the secondary file system because URI is not " +
- "provided.");
+ String secUri = props.get(IgfsConst.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
try {
- secondaryUri = new URI(secUri);
-
- URL secondaryCfgUrl = U.resolveIgniteUrl(secConfPath);
-
- if (secondaryCfgUrl == null)
- throw new IOException("Failed to resolve secondary file system config URL: " + secConfPath);
+ SecondaryFileSystemProvider secondaryFileSystemProvider =
+ new SecondaryFileSystemProvider(secUri, secConfPath, true);
- Configuration conf = new Configuration();
-
- conf.addResource(secondaryCfgUrl);
-
- String prop = String.format("fs.%s.impl.disable.cache", secondaryUri.getScheme());
-
- conf.setBoolean(prop, true);
-
- secondaryFs = AbstractFileSystem.get(secondaryUri, conf);
- }
- catch (URISyntaxException ignore) {
- throw new IOException("Failed to resolve secondary file system URI: " + secUri);
+ secondaryFs = secondaryFileSystemProvider.getAbstractFileSystem();
+ secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri();
}
catch (IOException e) {
throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
[2/2] incubator-ignite git commit: [IGNITE-349]: secondary filesystem
configuration made optional. The logic of the filesystem construction made
common,
and is now reused in all places where used. Constants also now specified in
single place and reused.
Posted by sb...@apache.org.
[IGNITE-349]: secondary filesystem configuration made optional. The logic of the filesystem construction made common, and is now reused in all places where used. Constants also now specified in single place and reused.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/e58cbfb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/e58cbfb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/e58cbfb4
Branch: refs/heads/ignite-349
Commit: e58cbfb4cc2f425ec2160af05ec532dea6b1c197
Parents: 51985d5
Author: iveselovskiy <iv...@gridgain.com>
Authored: Mon Mar 2 21:21:34 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Mon Mar 2 21:21:34 2015 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/igfs/IgfsConst.java | 28 ++++++++++++++++++++
.../visor/node/VisorIgfsConfiguration.java | 9 ++-----
2 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/e58cbfb4/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
new file mode 100644
index 0000000..859415b
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs;
+
+/**
+ * Constants related to Igfs secondary filesystem configuration.
+ */
+public class IgfsConst {
+ /** Property name for path to Hadoop configuration. */
+ public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
+ /** Property name for URI of file system. */
+ public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/e58cbfb4/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 056ac7f..3ec9d87 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -31,11 +31,6 @@ import static org.apache.ignite.internal.visor.util.VisorTaskUtils.*;
* Data transfer object for IGFS configuration properties.
*/
public class VisorIgfsConfiguration implements Serializable {
- /** Property name for path to Hadoop configuration. */
- public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
-
- /** Property name for URI of file system. */
- public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
/** */
private static final long serialVersionUID = 0L;
@@ -142,8 +137,8 @@ public class VisorIgfsConfiguration implements Serializable {
if (secFs != null) {
Map<String, String> props = secFs.properties();
- cfg.secondaryHadoopFileSysUri = props.get(SECONDARY_FS_URI);
- cfg.secondaryHadoopFileSysCfgPath = props.get(SECONDARY_FS_CONFIG_PATH);
+ cfg.secondaryHadoopFileSysUri = props.get(IgfsConst.SECONDARY_FS_URI);
+ cfg.secondaryHadoopFileSysCfgPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
}
cfg.dfltMode = igfs.getDefaultMode();