You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by sb...@apache.org on 2015/03/03 13:58:26 UTC
[50/50] incubator-ignite git commit: [IGNITE-349]: corrections by
review results.
[IGNITE-349]: corrections by review results.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/0a62a799
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/0a62a799
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/0a62a799
Branch: refs/heads/ignite-349
Commit: 0a62a799de9067c27eaf97782bb17d6bbe46d15a
Parents: 3015458
Author: iveselovskiy <iv...@gridgain.com>
Authored: Tue Mar 3 15:56:33 2015 +0300
Committer: iveselovskiy <iv...@gridgain.com>
Committed: Tue Mar 3 15:56:33 2015 +0300
----------------------------------------------------------------------
.../java/org/apache/ignite/igfs/IgfsConst.java | 28 ----
.../ignite/internal/processors/igfs/IgfsEx.java | 10 ++
.../visor/node/VisorIgfsConfiguration.java | 5 +-
.../hadoop/IgfsHadoopFileSystemWrapper.java | 14 +-
.../hadoop/SecondaryFileSystemProvider.java | 152 -------------------
.../igfs/hadoop/v1/IgfsHadoopFileSystem.java | 12 +-
.../igfs/hadoop/v2/IgfsHadoopFileSystem.java | 12 +-
.../hadoop/SecondaryFileSystemProvider.java | 112 ++++++++++++++
8 files changed, 148 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
deleted file mode 100644
index 859415b..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsConst.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-/**
- * Constants related to Igfs secondary filesystem configuration.
- */
-public class IgfsConst {
- /** Property name for path to Hadoop configuration. */
- public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
- /** Property name for URI of file system. */
- public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index e067e78..f247fba 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -30,6 +30,16 @@ import java.net.*;
*/
public interface IgfsEx extends IgniteFs {
/**
+ * Property name for path to Hadoop configuration.
+ */
+ String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
+
+ /**
+ * Property name for URI of file system.
+ */
+ String SECONDARY_FS_URI = "SECONDARY_FS_URI";
+
+ /**
* Stops IGFS cleaning all used resources.
*/
public void stop();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 3ec9d87..b8fa57c 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.visor.node;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
@@ -137,8 +138,8 @@ public class VisorIgfsConfiguration implements Serializable {
if (secFs != null) {
Map<String, String> props = secFs.properties();
- cfg.secondaryHadoopFileSysUri = props.get(IgfsConst.SECONDARY_FS_URI);
- cfg.secondaryHadoopFileSysCfgPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
+ cfg.secondaryHadoopFileSysUri = props.get(IgfsEx.SECONDARY_FS_URI);
+ cfg.secondaryHadoopFileSysCfgPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH);
}
cfg.dfltMode = igfs.getDefaultMode();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
index 2577516..fee4681 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/IgfsHadoopFileSystemWrapper.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.ipc.*;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.internal.igfs.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.jetbrains.annotations.*;
@@ -63,10 +64,17 @@ public class IgfsHadoopFileSystemWrapper implements Igfs, AutoCloseable {
public IgfsHadoopFileSystemWrapper(@Nullable String uri, @Nullable String cfgPath) throws IgniteCheckedException {
try {
SecondaryFileSystemProvider secondaryFileSystemProvider =
- new SecondaryFileSystemProvider(uri, cfgPath, false); // TODO: may be disableCaching == true there?
+ new SecondaryFileSystemProvider(uri, cfgPath);
- fileSys = secondaryFileSystemProvider.getFileSystem();
- secondaryFileSystemProvider.exportProperties(props);
+ fileSys = secondaryFileSystemProvider.createFileSystem();
+
+ uri = secondaryFileSystemProvider.secondaryFsUri().toString();
+
+ if (!uri.endsWith("/"))
+ uri += "/";
+
+ props.put(IgfsEx.SECONDARY_FS_CONFIG_PATH, cfgPath);
+ props.put(IgfsEx.SECONDARY_FS_URI, uri);
}
catch (IOException e) {
throw new IgniteCheckedException(e);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
deleted file mode 100644
index 81a12ea..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/SecondaryFileSystemProvider.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.hadoop;
-
-import com.sun.istack.NotNull;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.AbstractFileSystem;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.igfs.IgfsConst;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.jetbrains.annotations.Nullable;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.Map;
-
-/**
- * Encapsulates logic of secondary filesystem creation.
- */
-public class SecondaryFileSystemProvider {
-
- private @NotNull final Configuration cfg = new Configuration();
- private @NotNull final URI secondaryFsUri;
- private @Nullable final String secConfPath;
- private final boolean disableCaching;
-
- /**
- * Creates new provider with given config parameters.
- * The configuration URL is optional. The filesystem URI must be specified either explicitly or in the
- * configuration provided.
- *
- * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified
- * as "fs.defaultFS" property in the provided configuration.
- * @param secConfPath the secondary Fs config URL (optional)
- * @param disableCaching 'true' to disable caching in the created Hadoop filesystem. Used to avoid double caching.
- * @throws IOException
- */
- public SecondaryFileSystemProvider(@Nullable String secUri, @Nullable String secConfPath,
- boolean disableCaching) throws IOException {
- if (secUri == null && secConfPath == null) {
- throw new IllegalArgumentException("Failed to get secondary file system: neither " +
- "secondaryUri nor secondaryConfigPath given.");
- }
-
- this.disableCaching = disableCaching;
-
- if (secConfPath != null) {
- URL url = U.resolveIgniteUrl(secConfPath);
- if (url == null) {
- // If secConfPath is given, it should be resolvable:
- throw new IllegalArgumentException("Failed to resolve secondary Fs " +
- "configuration path [" + secConfPath + "]");
- }
- cfg.addResource(url);
- }
-
- this.secConfPath = secConfPath;
-
- try {
- // if secondary fs URI is not given explicitly, try to take it from the configuration:
- secondaryFsUri = (secUri == null) ? FileSystem.getDefaultUri(cfg) : new URI(secUri);
- } catch (URISyntaxException use) {
- throw new IOException("Failed to resolve secondary file system URI: " + secUri);
- }
-
- if (secondaryFsUri == null) {
- throw new IllegalArgumentException("Failed to get secondary file system URI: it is neither given " +
- "explicitly nor specified in the configuration ["+secConfPath+"]");
- }
- }
-
- /**
- * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
- * @throws IOException
- */
- public @NotNull FileSystem getFileSystem() throws IOException {
- if (disableCaching) {
- disableCaching();
- }
-
- FileSystem fileSys = FileSystem.get(secondaryFsUri, cfg);
-
- return fileSys;
- }
-
- /**
- * Exports properties of the filesystem to the given map.
- * @param props
- */
- public void exportProperties(@NotNull Map<String,String> props) {
- props.put(IgfsConst.SECONDARY_FS_CONFIG_PATH, secConfPath);
- props.put(IgfsConst.SECONDARY_FS_URI, getSecondaryFsUriStr());
- }
-
- /**
- * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs.
- * @throws IOException
- */
- public @NotNull AbstractFileSystem getAbstractFileSystem() throws IOException {
- if (disableCaching) {
- disableCaching();
- }
-
- AbstractFileSystem secondaryFs = AbstractFileSystem.get(secondaryFsUri, cfg);
-
- return secondaryFs;
- }
-
- private void disableCaching() {
- String prop = String.format("fs.%s.impl.disable.cache", secondaryFsUri.getScheme());
-
- cfg.setBoolean(prop, true);
- }
-
- /**
- * @return the secondary fs URI, never null.
- */
- public @NotNull URI getSecondaryFsUri() {
- return secondaryFsUri;
- }
-
- /**
- *
- * @return "slashified" secondary Fs URI in String form, never null.
- */
- private @NotNull String getSecondaryFsUriStr() {
- String uri = secondaryFsUri.toString();
-
- if (!uri.endsWith("/"))
- uri += "/";
-
- return uri;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
index cdfe1a1..1f7b87e 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v1/IgfsHadoopFileSystem.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.util.*;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.hadoop.*;
import org.apache.ignite.internal.igfs.common.*;
import org.apache.ignite.internal.igfs.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -282,15 +282,15 @@ public class IgfsHadoopFileSystem extends FileSystem {
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgfsConst.SECONDARY_FS_URI);
- String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
+ String secUri = props.get(IgfsEx.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH);
try {
SecondaryFileSystemProvider secondaryFileSystemProvider =
- new SecondaryFileSystemProvider(secUri, secConfPath, true);
+ new SecondaryFileSystemProvider(secUri, secConfPath);
- secondaryFs = secondaryFileSystemProvider.getFileSystem();
- secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri();
+ secondaryFs = secondaryFileSystemProvider.createFileSystem();
+ secondaryUri = secondaryFileSystemProvider.secondaryFsUri();
}
catch (IOException e) {
if (!mgmt)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
index 691705b..cb35bec 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/igfs/hadoop/v2/IgfsHadoopFileSystem.java
@@ -26,9 +26,9 @@ import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.util.*;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.hadoop.*;
import org.apache.ignite.internal.igfs.common.*;
import org.apache.ignite.internal.igfs.hadoop.*;
+import org.apache.ignite.internal.processors.hadoop.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -277,15 +277,15 @@ public class IgfsHadoopFileSystem extends AbstractFileSystem implements Closeabl
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgfsConst.SECONDARY_FS_URI);
- String secConfPath = props.get(IgfsConst.SECONDARY_FS_CONFIG_PATH);
+ String secUri = props.get(IgfsEx.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgfsEx.SECONDARY_FS_CONFIG_PATH);
try {
SecondaryFileSystemProvider secondaryFileSystemProvider =
- new SecondaryFileSystemProvider(secUri, secConfPath, true);
+ new SecondaryFileSystemProvider(secUri, secConfPath);
- secondaryFs = secondaryFileSystemProvider.getAbstractFileSystem();
- secondaryUri = secondaryFileSystemProvider.getSecondaryFsUri();
+ secondaryFs = secondaryFileSystemProvider.createAbstractFileSystem();
+ secondaryUri = secondaryFileSystemProvider.secondaryFsUri();
}
catch (IOException e) {
throw new IOException("Failed to connect to the secondary file system: " + secUri, e);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/0a62a799/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
new file mode 100644
index 0000000..a63242c
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/SecondaryFileSystemProvider.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+
+/**
+ * Encapsulates logic of secondary filesystem creation.
+ */
+public class SecondaryFileSystemProvider {
+ /**
+ * Configuration of the secondary filesystem, never null.
+ */
+ private final Configuration cfg = new Configuration();
+
+ /**
+ * The secondary filesystem URI, never null.
+ */
+ private final URI secondaryFsUri;
+
+ /**
+ * Creates new provider with given config parameters. The configuration URL is optional. The filesystem URI must be
+ * specified either explicitly or in the configuration provided.
+ *
+ * @param secUri the secondary Fs URI (optional). If not given explicitly, it must be specified as "fs.defaultFS"
+ * property in the provided configuration.
+ * @param secConfPath the secondary Fs config URL (optional)
+ * @throws IOException
+ */
+ public SecondaryFileSystemProvider(final @Nullable String secUri,
+ final @Nullable String secConfPath) throws IOException {
+ if (secConfPath != null) {
+ URL url = U.resolveIgniteUrl(secConfPath);
+ if (url == null) {
+ // If secConfPath is given, it should be resolvable:
+ throw new IllegalArgumentException("Failed to resolve secondary Fs " +
+ "configuration path [" + secConfPath + "]");
+ }
+ cfg.addResource(url);
+ }
+
+ // if secondary fs URI is not given explicitly, try to get it from the configuration:
+ if (secUri == null) {
+ secondaryFsUri = FileSystem.getDefaultUri(cfg);
+ }
+ else {
+ try {
+ secondaryFsUri = new URI(secUri);
+ }
+ catch (URISyntaxException use) {
+ throw new IOException("Failed to resolve secondary file system URI: " + secUri);
+ }
+ }
+
+ if (secondaryFsUri == null) {
+ throw new IllegalArgumentException("Failed to get secondary file system URI: it is neither given " +
+ "explicitly nor specified in the configuration: " + secConfPath);
+ }
+
+ // Disable caching:
+ String prop = String.format("fs.%s.impl.disable.cache", secondaryFsUri.getScheme());
+ cfg.setBoolean(prop, true);
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.FileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ public FileSystem createFileSystem() throws IOException {
+ FileSystem fileSys = FileSystem.get(secondaryFsUri, cfg);
+
+ return fileSys;
+ }
+
+ /**
+ * @return {@link org.apache.hadoop.fs.AbstractFileSystem} instance for this secondary Fs.
+ * @throws IOException
+ */
+ public AbstractFileSystem createAbstractFileSystem() throws IOException {
+ AbstractFileSystem secondaryFs = AbstractFileSystem.get(secondaryFsUri, cfg);
+
+ return secondaryFs;
+ }
+
+ /**
+ * @return the secondary fs URI, never null.
+ */
+ public URI secondaryFsUri() {
+ return secondaryFsUri;
+ }
+}