You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by vo...@apache.org on 2015/03/04 10:27:13 UTC
[1/6] incubator-ignite git commit: # IGNITE-386: WIP.
Repository: incubator-ignite
Updated Branches:
refs/heads/ignite-386 28fad1854 -> d83c88af0
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
index 364eaa1..c1b36a6 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsEventsAbstractSelfTest.java
@@ -40,7 +40,7 @@ import static org.apache.ignite.events.EventType.*;
import static org.apache.ignite.testframework.GridTestUtils.*;
/**
- * Tests events, generated by {@link org.apache.ignite.IgniteFs} implementation.
+ * Tests events, generated by {@link org.apache.ignite.IgniteFileSystem} implementation.
*/
public abstract class IgfsEventsAbstractSelfTest extends GridCommonAbstractTest {
/** IGFS. */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
index 99a40c4..b51db0c 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerSelfTest.java
@@ -36,7 +36,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
* @throws Exception If failed.
*/
public void testReadFragmentizing() throws Exception {
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
IgfsPath path = new IgfsPath("/someFile");
@@ -108,7 +108,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
int fileSize = 30 * IGFS_GROUP_SIZE * IGFS_BLOCK_SIZE;
while (written < fileSize) {
- IgniteFs igfs = grid(igfsIdx).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(igfsIdx).fileSystem("igfs");
try (IgfsOutputStream out = igfs.append(path, true)) {
byte[] data = new byte[chunkSize];
@@ -131,7 +131,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
}
}
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
try (IgfsInputStream in = igfs.open(path)) {
i = 0;
@@ -180,7 +180,7 @@ public class IgfsFragmentizerSelfTest extends IgfsFragmentizerAbstractSelfTest {
int fileSize = 50 * IGFS_GROUP_SIZE * IGFS_BLOCK_SIZE;
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
byte[] chunk = new byte[chunkSize];
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
index 59faa42..cb7b63a 100644
--- a/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/igfs/IgfsFragmentizerTopologySelfTest.java
@@ -33,7 +33,7 @@ public class IgfsFragmentizerTopologySelfTest extends IgfsFragmentizerAbstractSe
try {
IgfsPath path = new IgfsPath("/someFile");
- IgniteFs igfs = grid(1).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(1).fileSystem("igfs");
try (IgfsOutputStream out = igfs.create(path, true)) {
for (int i = 0; i < 10 * IGFS_GROUP_SIZE; i++)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
index f0e2c70..a2e1c81 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
@@ -65,7 +65,7 @@ public class GridCacheIgfsPerBlockLruEvictionPolicySelfTest extends IgfsCommonAb
private static IgfsImpl igfsPrimary;
/** Secondary IGFS instance. */
- private static IgniteFs secondaryFs;
+ private static IgniteFileSystem secondaryFs;
/** Primary file system data cache. */
private static GridCacheAdapter<IgfsBlockKey, byte[]> dataCache;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index a7e6780..7bc7694 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -21,6 +21,7 @@ import org.apache.ignite.*;
import org.apache.ignite.cache.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.lang.*;
import org.apache.ignite.internal.util.typedef.*;
@@ -37,7 +38,7 @@ import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.cache.CacheAtomicityMode.*;
import static org.apache.ignite.cache.CacheMemoryMode.*;
import static org.apache.ignite.cache.CacheMode.*;
@@ -181,7 +182,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable Igfs secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
IgfsConfiguration igfsCfg = new IgfsConfiguration();
igfsCfg.setDataCacheName("dataCache");
@@ -2137,7 +2138,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param files Files.
* @throws Exception If failed.
*/
- public static void create(Igfs igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
+ public static void create(IgfsSecondaryFileSystem igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
throws Exception {
if (dirs != null) {
for (IgfsPath dir : dirs)
@@ -2162,7 +2163,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws IOException In case of IO exception.
*/
- protected static void createFile(Igfs igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
+ protected static void createFile(IgfsSecondaryFileSystem igfs, IgfsPath file, boolean overwrite, @Nullable byte[]... chunks)
throws IOException {
OutputStream os = null;
@@ -2188,7 +2189,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void createFile(IgniteFs igfs, IgfsPath file, boolean overwrite, long blockSize,
+ protected void createFile(IgniteFileSystem igfs, IgfsPath file, boolean overwrite, long blockSize,
@Nullable byte[]... chunks) throws Exception {
IgfsOutputStream os = null;
@@ -2212,7 +2213,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void appendFile(IgniteFs igfs, IgfsPath file, @Nullable byte[]... chunks)
+ protected void appendFile(IgniteFileSystem igfs, IgfsPath file, @Nullable byte[]... chunks)
throws Exception {
IgfsOutputStream os = null;
@@ -2248,7 +2249,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfs IGFS.
* @param file File.
*/
- public static void awaitFileClose(Igfs igfs, IgfsPath file) {
+ public static void awaitFileClose(IgfsSecondaryFileSystem igfs, IgfsPath file) {
try {
igfs.update(file, Collections.singletonMap("prop", "val"));
}
@@ -2418,7 +2419,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfsSecondary Second IGFS.
* @throws Exception If failed.
*/
- protected void clear(IgniteFs igfs, IgniteFs igfsSecondary) throws Exception {
+ protected void clear(IgniteFileSystem igfs, IgniteFileSystem igfsSecondary) throws Exception {
clear(igfs);
if (dual)
@@ -2431,7 +2432,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param igfs IGFS.
* @throws Exception If failed.
*/
- public static void clear(IgniteFs igfs) throws Exception {
+ public static void clear(IgniteFileSystem igfs) throws Exception {
Field workerMapFld = IgfsImpl.class.getDeclaredField("workerMap");
workerMapFld.setAccessible(true);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 327d7fa..1566cca 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -28,7 +28,7 @@ import java.io.*;
import java.util.*;
import java.util.concurrent.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.igfs.IgfsMode.*;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
index be5afe7..d6caf27 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
@@ -56,10 +56,10 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
private static final TcpDiscoveryIpFinder IP_FINDER = new TcpDiscoveryVmIpFinder(true);
/** Primary IGFS instances. */
- private static IgniteFs[] igfsPrimary;
+ private static IgniteFileSystem[] igfsPrimary;
/** Secondary IGFS instance. */
- private static IgniteFs igfsSecondary;
+ private static IgniteFileSystem igfsSecondary;
/** Primary file system block size. */
public static final int PRIMARY_BLOCK_SIZE = 512;
@@ -84,7 +84,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
private void startPrimary() throws Exception {
- igfsPrimary = new IgniteFs[NODES_CNT];
+ igfsPrimary = new IgniteFileSystem[NODES_CNT];
for (int i = 0; i < NODES_CNT; i++) {
Ignite g = G.start(primaryConfiguration(i));
@@ -209,7 +209,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
/** @throws Exception If failed. */
public void testMetrics() throws Exception {
- IgniteFs fs = igfsPrimary[0];
+ IgniteFileSystem fs = igfsPrimary[0];
assertNotNull(fs);
@@ -349,7 +349,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
/** @throws Exception If failed. */
public void testMultipleClose() throws Exception {
- IgniteFs fs = igfsPrimary[0];
+ IgniteFileSystem fs = igfsPrimary[0];
IgfsOutputStream out = fs.create(new IgfsPath("/file"), false);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
index ead7511..135e543 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorSelfTest.java
@@ -59,7 +59,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
protected final SecureRandom rnd = new SecureRandom();
/** File system. */
- protected IgniteFs igfs;
+ protected IgniteFileSystem igfs;
/** Meta cache. */
private GridCache<Object, Object> metaCache;
@@ -157,7 +157,7 @@ public class IgfsProcessorSelfTest extends IgfsCommonAbstractTest {
/** @throws Exception If failed. */
public void testigfsEnabled() throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName());
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName());
assertNotNull(igfs);
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
index 054665d..fa8bf89 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
@@ -74,7 +74,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
public static final int ASSERT_RETRY_INTERVAL = 100;
/** File system to test. */
- private IgniteFs fs;
+ private IgniteFileSystem fs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
@@ -229,12 +229,12 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
IgfsPath path = new IgfsPath("/file");
try {
- IgniteFs fs0 = grid(0).fileSystem("igfs");
- IgniteFs fs1 = grid(1).fileSystem("igfs");
- IgniteFs fs2 = grid(2).fileSystem("igfs");
+ IgniteFileSystem fs0 = grid(0).fileSystem("igfs");
+ IgniteFileSystem fs1 = grid(1).fileSystem("igfs");
+ IgniteFileSystem fs2 = grid(2).fileSystem("igfs");
try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE,
- F.asMap(IgniteFs.PROP_PREFER_LOCAL_WRITES, "true"))) {
+ F.asMap(IgniteFileSystem.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
index 5b7a636..2f096d1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsTaskSelfTest.java
@@ -68,7 +68,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
private static final int REPEAT_CNT = 10;
/** IGFS. */
- private static IgniteFs igfs;
+ private static IgniteFileSystem igfs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
@@ -172,7 +172,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
assertFalse(igfs.isAsync());
- IgniteFs igfsAsync = igfs.withAsync();
+ IgniteFileSystem igfsAsync = igfs.withAsync();
assertTrue(igfsAsync.isAsync());
@@ -269,7 +269,7 @@ public class IgfsTaskSelfTest extends IgfsCommonAbstractTest {
private ComputeJobContext ctx;
/** {@inheritDoc} */
- @Override public Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in)
+ @Override public Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in)
throws IOException {
assert ignite != null;
assert ses != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
index 05de61a..e7c819b 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/split/IgfsAbstractRecordResolverSelfTest.java
@@ -46,7 +46,7 @@ public class IgfsAbstractRecordResolverSelfTest extends GridCommonAbstractTest {
private final TcpDiscoveryIpFinder ipFinder = new TcpDiscoveryVmIpFinder(true);
/** IGFS. */
- protected static IgniteFs igfs;
+ protected static IgniteFileSystem igfs;
/** {@inheritDoc} */
@Override protected void beforeTestsStarted() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
index 58478d3..220d5d6 100644
--- a/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
+++ b/modules/core/src/test/java/org/apache/ignite/testframework/junits/IgniteMock.java
@@ -176,12 +176,12 @@ public class IgniteMock implements Ignite {
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
return null;
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
new file mode 100644
index 0000000..65d1901
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -0,0 +1,415 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.hadoop.fs;
+
+import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.ipc.*;
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.processors.hadoop.igfs.*;
+import org.apache.ignite.internal.processors.igfs.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.net.*;
+import java.util.*;
+
+/**
+ * Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as
+ * {@link org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem}.
+ */
+public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSystem, AutoCloseable {
+ /** Property name for path to Hadoop configuration. */
+ public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
+
+ /** Property name for URI of file system. */
+ public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
+
+ /** Hadoop file system. */
+ private final FileSystem fileSys;
+
+ /** Properties of file system */
+ private final Map<String, String> props = new HashMap<>();
+
+ /**
+ * Constructor.
+ *
+ * @param uri URI of file system.
+ * @param cfgPath Additional path to Hadoop configuration.
+ * @throws IgniteCheckedException In case of error.
+ */
+ public IgniteHadoopIgfsSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath)
+ throws IgniteCheckedException {
+ Configuration cfg = new Configuration();
+
+ if (cfgPath != null)
+ cfg.addResource(U.resolveIgniteUrl(cfgPath));
+
+ try {
+ fileSys = uri == null ? FileSystem.get(cfg) : FileSystem.get(new URI(uri), cfg);
+ }
+ catch (IOException | URISyntaxException e) {
+ throw new IgniteCheckedException(e);
+ }
+
+ uri = fileSys.getUri().toString();
+
+ if (!uri.endsWith("/"))
+ uri += "/";
+
+ props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
+ props.put(SECONDARY_FS_URI, uri);
+ }
+
+ /**
+ * Convert IGFS path into Hadoop path.
+ *
+ * @param path IGFS path.
+ * @return Hadoop path.
+ */
+ private Path convert(IgfsPath path) {
+ URI uri = fileSys.getUri();
+
+ return new Path(uri.getScheme(), uri.getAuthority(), path.toString());
+ }
+
+ /**
+ * Heuristically checks if exception was caused by invalid HDFS version and returns appropriate exception.
+ *
+ * @param e Exception to check.
+ * @param detailMsg Detailed error message.
+ * @return Appropriate exception.
+ */
+ private IgfsException handleSecondaryFsError(IOException e, String detailMsg) {
+ boolean wrongVer = X.hasCause(e, RemoteException.class) ||
+ (e.getMessage() != null && e.getMessage().contains("Failed on local"));
+
+ IgfsException igfsErr = !wrongVer ? cast(detailMsg, e) :
+ new IgfsInvalidHdfsVersionException("HDFS version you are connecting to differs from local " +
+ "version.", e);
+
+ return igfsErr;
+ }
+
+ /**
+ * Cast IO exception to IGFS exception.
+ *
+ * @param e IO exception.
+ * @return IGFS exception.
+ */
+ public static IgfsException cast(String msg, IOException e) {
+ if (e instanceof FileNotFoundException)
+ return new IgfsFileNotFoundException(e);
+ else if (e instanceof ParentNotDirectoryException)
+ return new IgfsParentNotDirectoryException(msg, e);
+ else if (e instanceof PathIsNotEmptyDirectoryException)
+ return new IgfsDirectoryNotEmptyException(e);
+ else if (e instanceof PathExistsException)
+ return new IgfsPathAlreadyExistsException(msg, e);
+ else
+ return new IgfsException(msg, e);
+ }
+
+ /**
+ * Convert Hadoop FileStatus properties to map.
+ *
+ * @param status File status.
+ * @return IGFS attributes.
+ */
+ private static Map<String, String> properties(FileStatus status) {
+ FsPermission perm = status.getPermission();
+
+ if (perm == null)
+ perm = FsPermission.getDefault();
+
+ return F.asMap(PROP_PERMISSION, String.format("%04o", perm.toShort()), PROP_USER_NAME, status.getOwner(),
+ PROP_GROUP_NAME, status.getGroup());
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(IgfsPath path) {
+ try {
+ return fileSys.exists(convert(path));
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
+ HadoopIgfsProperties props0 = new HadoopIgfsProperties(props);
+
+ try {
+ if (props0.userName() != null || props0.groupName() != null)
+ fileSys.setOwner(convert(path), props0.userName(), props0.groupName());
+
+ if (props0.permission() != null)
+ fileSys.setPermission(convert(path), props0.permission());
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]");
+ }
+
+ //Result is not used in case of secondary FS.
+ return null;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rename(IgfsPath src, IgfsPath dest) {
+ // Delegate to the secondary file system.
+ try {
+ if (!fileSys.rename(convert(src), convert(dest)))
+ throw new IgfsException("Failed to rename (secondary file system returned false) " +
+ "[src=" + src + ", dest=" + dest + ']');
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to rename file [src=" + src + ", dest=" + dest + ']');
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(IgfsPath path, boolean recursive) {
+ try {
+ return fileSys.delete(convert(path), recursive);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path) {
+ try {
+ if (!fileSys.mkdirs(convert(path)))
+ throw new IgniteException("Failed to make directories [path=" + path + "]");
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
+ try {
+ if (!fileSys.mkdirs(convert(path), new HadoopIgfsProperties(props).permission()))
+ throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]");
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + ", props=" + props + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
+ try {
+ FileStatus[] statuses = fileSys.listStatus(convert(path));
+
+ if (statuses == null)
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+
+ Collection<IgfsPath> res = new ArrayList<>(statuses.length);
+
+ for (FileStatus status : statuses)
+ res.add(new IgfsPath(path, status.getPath().getName()));
+
+ return res;
+ }
+ catch (FileNotFoundException ignored) {
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
+ try {
+ FileStatus[] statuses = fileSys.listStatus(convert(path));
+
+ if (statuses == null)
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+
+ Collection<IgfsFile> res = new ArrayList<>(statuses.length);
+
+ for (FileStatus status : statuses) {
+ IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status)) :
+ new IgfsFileInfo((int)status.getBlockSize(), status.getLen(), null, null, false,
+ properties(status));
+
+ res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
+ }
+
+ return res;
+ }
+ catch (FileNotFoundException ignored) {
+ throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystemReader open(IgfsPath path, int bufSize) {
+ return new HadoopIgfsSecondaryFileSystemReader(fileSys, convert(path), bufSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, boolean overwrite) {
+ try {
+ return fileSys.create(convert(path), overwrite);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) {
+ HadoopIgfsProperties props0 =
+ new HadoopIgfsProperties(props != null ? props : Collections.<String, String>emptyMap());
+
+ try {
+ return fileSys.create(convert(path), props0.permission(), overwrite, bufSize, (short)replication, blockSize,
+ null);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props +
+ ", overwrite=" + overwrite + ", bufSize=" + bufSize + ", replication=" + replication +
+ ", blockSize=" + blockSize + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) {
+ try {
+ return fileSys.append(convert(path), bufSize);
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(final IgfsPath path) {
+ try {
+ final FileStatus status = fileSys.getFileStatus(convert(path));
+
+ if (status == null)
+ return null;
+
+ final Map<String, String> props = properties(status);
+
+ return new IgfsFile() {
+ @Override public IgfsPath path() {
+ return path;
+ }
+
+ @Override public boolean isFile() {
+ return status.isFile();
+ }
+
+ @Override public boolean isDirectory() {
+ return status.isDirectory();
+ }
+
+ @Override public int blockSize() {
+ return (int)status.getBlockSize();
+ }
+
+ @Override public long groupBlockSize() {
+ return status.getBlockSize();
+ }
+
+ @Override public long accessTime() {
+ return status.getAccessTime();
+ }
+
+ @Override public long modificationTime() {
+ return status.getModificationTime();
+ }
+
+ @Override public String property(String name) throws IllegalArgumentException {
+ String val = props.get(name);
+
+ if (val == null)
+ throw new IllegalArgumentException("File property not found [path=" + path + ", name=" + name + ']');
+
+ return val;
+ }
+
+ @Nullable @Override public String property(String name, @Nullable String dfltVal) {
+ String val = props.get(name);
+
+ return val == null ? dfltVal : val;
+ }
+
+ @Override public long length() {
+ return status.getLen();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> properties() {
+ return props;
+ }
+ };
+
+ }
+ catch (FileNotFoundException ignore) {
+ return null;
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long usedSpaceSize() {
+ try {
+ return fileSys.getContentSummary(new Path(fileSys.getUri())).getSpaceConsumed();
+ }
+ catch (IOException e) {
+ throw handleSecondaryFsError(e, "Failed to get used space size of file system.");
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Nullable @Override public Map<String, String> properties() {
+ return props;
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() throws IgniteCheckedException {
+ try {
+ fileSys.close();
+ }
+ catch (IOException e) {
+ throw new IgniteCheckedException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopSecondaryFileSystem.java
deleted file mode 100644
index 44e2c20..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopSecondaryFileSystem.java
+++ /dev/null
@@ -1,413 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.hadoop.fs;
-
-import org.apache.hadoop.conf.*;
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.permission.*;
-import org.apache.hadoop.ipc.*;
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.processors.hadoop.igfs.*;
-import org.apache.ignite.internal.processors.igfs.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.net.*;
-import java.util.*;
-
-/**
- * Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as {@link org.apache.ignite.igfs.Igfs}.
- */
-public class IgniteHadoopSecondaryFileSystem implements Igfs, AutoCloseable {
- /** Property name for path to Hadoop configuration. */
- public static final String SECONDARY_FS_CONFIG_PATH = "SECONDARY_FS_CONFIG_PATH";
-
- /** Property name for URI of file system. */
- public static final String SECONDARY_FS_URI = "SECONDARY_FS_URI";
-
- /** Hadoop file system. */
- private final FileSystem fileSys;
-
- /** Properties of file system */
- private final Map<String, String> props = new HashMap<>();
-
- /**
- * Constructor.
- *
- * @param uri URI of file system.
- * @param cfgPath Additional path to Hadoop configuration.
- * @throws IgniteCheckedException In case of error.
- */
- public IgniteHadoopSecondaryFileSystem(@Nullable String uri, @Nullable String cfgPath)
- throws IgniteCheckedException {
- Configuration cfg = new Configuration();
-
- if (cfgPath != null)
- cfg.addResource(U.resolveIgniteUrl(cfgPath));
-
- try {
- fileSys = uri == null ? FileSystem.get(cfg) : FileSystem.get(new URI(uri), cfg);
- }
- catch (IOException | URISyntaxException e) {
- throw new IgniteCheckedException(e);
- }
-
- uri = fileSys.getUri().toString();
-
- if (!uri.endsWith("/"))
- uri += "/";
-
- props.put(SECONDARY_FS_CONFIG_PATH, cfgPath);
- props.put(SECONDARY_FS_URI, uri);
- }
-
- /**
- * Convert IGFS path into Hadoop path.
- *
- * @param path IGFS path.
- * @return Hadoop path.
- */
- private Path convert(IgfsPath path) {
- URI uri = fileSys.getUri();
-
- return new Path(uri.getScheme(), uri.getAuthority(), path.toString());
- }
-
- /**
- * Heuristically checks if exception was caused by invalid HDFS version and returns appropriate exception.
- *
- * @param e Exception to check.
- * @param detailMsg Detailed error message.
- * @return Appropriate exception.
- */
- private IgfsException handleSecondaryFsError(IOException e, String detailMsg) {
- boolean wrongVer = X.hasCause(e, RemoteException.class) ||
- (e.getMessage() != null && e.getMessage().contains("Failed on local"));
-
- IgfsException igfsErr = !wrongVer ? cast(detailMsg, e) :
- new IgfsInvalidHdfsVersionException("HDFS version you are connecting to differs from local " +
- "version.", e);
-
- return igfsErr;
- }
-
- /**
- * Cast IO exception to IGFS exception.
- *
- * @param e IO exception.
- * @return IGFS exception.
- */
- public static IgfsException cast(String msg, IOException e) {
- if (e instanceof FileNotFoundException)
- return new IgfsFileNotFoundException(e);
- else if (e instanceof ParentNotDirectoryException)
- return new IgfsParentNotDirectoryException(msg, e);
- else if (e instanceof PathIsNotEmptyDirectoryException)
- return new IgfsDirectoryNotEmptyException(e);
- else if (e instanceof PathExistsException)
- return new IgfsPathAlreadyExistsException(msg, e);
- else
- return new IgfsException(msg, e);
- }
-
- /**
- * Convert Hadoop FileStatus properties to map.
- *
- * @param status File status.
- * @return IGFS attributes.
- */
- private static Map<String, String> properties(FileStatus status) {
- FsPermission perm = status.getPermission();
-
- if (perm == null)
- perm = FsPermission.getDefault();
-
- return F.asMap(PROP_PERMISSION, String.format("%04o", perm.toShort()), PROP_USER_NAME, status.getOwner(),
- PROP_GROUP_NAME, status.getGroup());
- }
-
- /** {@inheritDoc} */
- @Override public boolean exists(IgfsPath path) {
- try {
- return fileSys.exists(convert(path));
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to check file existence [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public IgfsFile update(IgfsPath path, Map<String, String> props) {
- HadoopIgfsProperties props0 = new HadoopIgfsProperties(props);
-
- try {
- if (props0.userName() != null || props0.groupName() != null)
- fileSys.setOwner(convert(path), props0.userName(), props0.groupName());
-
- if (props0.permission() != null)
- fileSys.setPermission(convert(path), props0.permission());
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to update file properties [path=" + path + "]");
- }
-
- //Result is not used in case of secondary FS.
- return null;
- }
-
- /** {@inheritDoc} */
- @Override public void rename(IgfsPath src, IgfsPath dest) {
- // Delegate to the secondary file system.
- try {
- if (!fileSys.rename(convert(src), convert(dest)))
- throw new IgfsException("Failed to rename (secondary file system returned false) " +
- "[src=" + src + ", dest=" + dest + ']');
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to rename file [src=" + src + ", dest=" + dest + ']');
- }
- }
-
- /** {@inheritDoc} */
- @Override public boolean delete(IgfsPath path, boolean recursive) {
- try {
- return fileSys.delete(convert(path), recursive);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to delete file [path=" + path + ", recursive=" + recursive + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path) {
- try {
- if (!fileSys.mkdirs(convert(path)))
- throw new IgniteException("Failed to make directories [path=" + path + "]");
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) {
- try {
- if (!fileSys.mkdirs(convert(path), new HadoopIgfsProperties(props).permission()))
- throw new IgniteException("Failed to make directories [path=" + path + ", props=" + props + "]");
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to make directories [path=" + path + ", props=" + props + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsPath> listPaths(IgfsPath path) {
- try {
- FileStatus[] statuses = fileSys.listStatus(convert(path));
-
- if (statuses == null)
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsPath> res = new ArrayList<>(statuses.length);
-
- for (FileStatus status : statuses)
- res.add(new IgfsPath(path, status.getPath().getName()));
-
- return res;
- }
- catch (FileNotFoundException ignored) {
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public Collection<IgfsFile> listFiles(IgfsPath path) {
- try {
- FileStatus[] statuses = fileSys.listStatus(convert(path));
-
- if (statuses == null)
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
-
- Collection<IgfsFile> res = new ArrayList<>(statuses.length);
-
- for (FileStatus status : statuses) {
- IgfsFileInfo fsInfo = status.isDirectory() ? new IgfsFileInfo(true, properties(status)) :
- new IgfsFileInfo((int)status.getBlockSize(), status.getLen(), null, null, false,
- properties(status));
-
- res.add(new IgfsFileImpl(new IgfsPath(path, status.getPath().getName()), fsInfo, 1));
- }
-
- return res;
- }
- catch (FileNotFoundException ignored) {
- throw new IgfsFileNotFoundException("Failed to list files (path not found): " + path);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to list statuses due to secondary file system exception: " + path);
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsReader open(IgfsPath path, int bufSize) {
- return new HadoopIgfsReader(fileSys, convert(path), bufSize);
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream create(IgfsPath path, boolean overwrite) {
- try {
- return fileSys.create(convert(path), overwrite);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", overwrite=" + overwrite + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) {
- HadoopIgfsProperties props0 =
- new HadoopIgfsProperties(props != null ? props : Collections.<String, String>emptyMap());
-
- try {
- return fileSys.create(convert(path), props0.permission(), overwrite, bufSize, (short)replication, blockSize,
- null);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to create file [path=" + path + ", props=" + props +
- ", overwrite=" + overwrite + ", bufSize=" + bufSize + ", replication=" + replication +
- ", blockSize=" + blockSize + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) {
- try {
- return fileSys.append(convert(path), bufSize);
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to append file [path=" + path + ", bufSize=" + bufSize + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public IgfsFile info(final IgfsPath path) {
- try {
- final FileStatus status = fileSys.getFileStatus(convert(path));
-
- if (status == null)
- return null;
-
- final Map<String, String> props = properties(status);
-
- return new IgfsFile() {
- @Override public IgfsPath path() {
- return path;
- }
-
- @Override public boolean isFile() {
- return status.isFile();
- }
-
- @Override public boolean isDirectory() {
- return status.isDirectory();
- }
-
- @Override public int blockSize() {
- return (int)status.getBlockSize();
- }
-
- @Override public long groupBlockSize() {
- return status.getBlockSize();
- }
-
- @Override public long accessTime() {
- return status.getAccessTime();
- }
-
- @Override public long modificationTime() {
- return status.getModificationTime();
- }
-
- @Override public String property(String name) throws IllegalArgumentException {
- String val = props.get(name);
-
- if (val == null)
- throw new IllegalArgumentException("File property not found [path=" + path + ", name=" + name + ']');
-
- return val;
- }
-
- @Nullable @Override public String property(String name, @Nullable String dfltVal) {
- String val = props.get(name);
-
- return val == null ? dfltVal : val;
- }
-
- @Override public long length() {
- return status.getLen();
- }
-
- /** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return props;
- }
- };
-
- }
- catch (FileNotFoundException ignore) {
- return null;
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to get file status [path=" + path + "]");
- }
- }
-
- /** {@inheritDoc} */
- @Override public long usedSpaceSize() {
- try {
- return fileSys.getContentSummary(new Path(fileSys.getUri())).getSpaceConsumed();
- }
- catch (IOException e) {
- throw handleSecondaryFsError(e, "Failed to get used space size of file system.");
- }
- }
-
- /** {@inheritDoc} */
- @Nullable @Override public Map<String, String> properties() {
- return props;
- }
-
- /** {@inheritDoc} */
- @Override public void close() throws IgniteCheckedException {
- try {
- fileSys.close();
- }
- catch (IOException e) {
- throw new IgniteCheckedException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 659561d..9261273 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -39,7 +39,7 @@ import java.net.*;
import java.util.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.configuration.IgfsConfiguration.*;
import static org.apache.ignite.igfs.IgfsMode.*;
import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
@@ -282,8 +282,8 @@ public class IgniteHadoopFileSystem extends FileSystem {
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgniteHadoopSecondaryFileSystem.SECONDARY_FS_URI);
- String secConfPath = props.get(IgniteHadoopSecondaryFileSystem.SECONDARY_FS_CONFIG_PATH);
+ String secUri = props.get(IgniteHadoopIgfsSecondaryFileSystem.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgniteHadoopIgfsSecondaryFileSystem.SECONDARY_FS_CONFIG_PATH);
if (secConfPath == null)
throw new IOException("Failed to connect to the secondary file system because configuration " +
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index 023faa4..fa1abb5 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -39,7 +39,7 @@ import java.net.*;
import java.util.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.configuration.IgfsConfiguration.*;
import static org.apache.ignite.igfs.IgfsMode.*;
import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
@@ -277,8 +277,8 @@ public class IgniteHadoopFileSystem extends AbstractFileSystem implements Closea
if (initSecondary) {
Map<String, String> props = paths.properties();
- String secUri = props.get(IgniteHadoopSecondaryFileSystem.SECONDARY_FS_URI);
- String secConfPath = props.get(IgniteHadoopSecondaryFileSystem.SECONDARY_FS_CONFIG_PATH);
+ String secUri = props.get(IgniteHadoopIgfsSecondaryFileSystem.SECONDARY_FS_URI);
+ String secConfPath = props.get(IgniteHadoopIgfsSecondaryFileSystem.SECONDARY_FS_CONFIG_PATH);
if (secConfPath == null)
throw new IOException("Failed to connect to the secondary file system because configuration " +
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
index f61c44b..16c9351 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsEndpoint.java
@@ -56,7 +56,7 @@ public class HadoopIgfsEndpoint {
*/
public static URI normalize(URI uri) throws IOException {
try {
- if (!F.eq(IgniteFs.IGFS_SCHEME, uri.getScheme()))
+ if (!F.eq(IgniteFileSystem.IGFS_SCHEME, uri.getScheme()))
throw new IOException("Failed to normalize UIR because it has non IGFS scheme: " + uri);
HadoopIgfsEndpoint endpoint = new HadoopIgfsEndpoint(uri.getAuthority());
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
index f99f14c..2cbf32f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
@@ -22,7 +22,7 @@ import org.apache.ignite.*;
import java.util.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
/**
* Hadoop file system properties.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsReader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsReader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsReader.java
deleted file mode 100644
index f410fae..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsReader.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.igfs;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
- * requested.
- * <p>
- * The class is expected to be used only from synchronized context and therefore is not tread-safe.
- */
-public class HadoopIgfsReader implements IgfsReader {
- /** Secondary file system. */
- private final FileSystem fs;
-
- /** Path to the file to open. */
- private final Path path;
-
- /** Buffer size. */
- private final int bufSize;
-
- /** Actual input stream. */
- private FSDataInputStream in;
-
- /** Cached error occurred during output stream open. */
- private IOException err;
-
- /** Flag indicating that the stream was already opened. */
- private boolean opened;
-
- /**
- * Constructor.
- *
- * @param fs Secondary file system.
- * @param path Path to the file to open.
- * @param bufSize Buffer size.
- */
- public HadoopIgfsReader(FileSystem fs, Path path, int bufSize) {
- assert fs != null;
- assert path != null;
-
- this.fs = fs;
- this.path = path;
- this.bufSize = bufSize;
- }
-
- /** Get input stream. */
- private PositionedReadable in() throws IOException {
- if (opened) {
- if (err != null)
- throw err;
- }
- else {
- opened = true;
-
- try {
- in = fs.open(path, bufSize);
-
- if (in == null)
- throw new IOException("Failed to open input stream (file system returned null): " + path);
- }
- catch (IOException e) {
- err = e;
-
- throw err;
- }
- }
-
- return in;
- }
-
- /**
- * Close wrapped input stream in case it was previously opened.
- */
- @Override public void close() {
- U.closeQuiet(in);
- }
-
- /** {@inheritDoc} */
- @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
- return in().read(pos, buf, off, len);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
new file mode 100644
index 0000000..15d5419
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
+ * requested.
+ * <p>
+ * The class is expected to be used only from synchronized context and therefore is not tread-safe.
+ */
+public class HadoopIgfsSecondaryFileSystemReader implements IgfsSecondaryFileSystemReader {
+ /** Secondary file system. */
+ private final FileSystem fs;
+
+ /** Path to the file to open. */
+ private final Path path;
+
+ /** Buffer size. */
+ private final int bufSize;
+
+ /** Actual input stream. */
+ private FSDataInputStream in;
+
+ /** Cached error occurred during output stream open. */
+ private IOException err;
+
+ /** Flag indicating that the stream was already opened. */
+ private boolean opened;
+
+ /**
+ * Constructor.
+ *
+ * @param fs Secondary file system.
+ * @param path Path to the file to open.
+ * @param bufSize Buffer size.
+ */
+ public HadoopIgfsSecondaryFileSystemReader(FileSystem fs, Path path, int bufSize) {
+ assert fs != null;
+ assert path != null;
+
+ this.fs = fs;
+ this.path = path;
+ this.bufSize = bufSize;
+ }
+
+ /** Get input stream. */
+ private PositionedReadable in() throws IOException {
+ if (opened) {
+ if (err != null)
+ throw err;
+ }
+ else {
+ opened = true;
+
+ try {
+ in = fs.open(path, bufSize);
+
+ if (in == null)
+ throw new IOException("Failed to open input stream (file system returned null): " + path);
+ }
+ catch (IOException e) {
+ err = e;
+
+ throw err;
+ }
+ }
+
+ return in;
+ }
+
+ /**
+ * Close wrapped input stream in case it was previously opened.
+ */
+ @Override public void close() {
+ U.closeQuiet(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
+ return in().read(pos, buf, off, len);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlanner.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlanner.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlanner.java
index b0a8ce96..d82bae7 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlanner.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/planner/HadoopDefaultMapReducePlanner.java
@@ -31,7 +31,7 @@ import org.jetbrains.annotations.*;
import java.util.*;
-import static org.apache.ignite.IgniteFs.*;
+import static org.apache.ignite.IgniteFileSystem.*;
/**
* Default map-reduce planner implementation.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
index 6eb53f9..d19a8ea 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/client/hadoop/HadoopClientProtocolSelfTest.java
@@ -152,7 +152,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* @throws Exception If failed.
*/
public void testJobCounters() throws Exception {
- IgniteFs igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
igfs.mkdirs(new IgfsPath(PATH_INPUT));
@@ -270,7 +270,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* @throws Exception If failed.
*/
public void checkJobSubmit(boolean noCombiners, boolean noReducers) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
+ IgniteFileSystem igfs = grid(0).fileSystem(HadoopAbstractSelfTest.igfsName);
igfs.mkdirs(new IgfsPath(PATH_INPUT));
@@ -402,7 +402,7 @@ public class HadoopClientProtocolSelfTest extends HadoopAbstractSelfTest {
* @throws Exception If failed.
*/
@SuppressWarnings("ConstantConditions")
- private static void dumpIgfs(IgniteFs igfs, IgfsPath path) throws Exception {
+ private static void dumpIgfs(IgniteFileSystem igfs, IgfsPath path) throws Exception {
IgfsFile file = igfs.info(path);
assert file != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
index 04f2d0c..b7aafff 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfs20FileSystemAbstractSelfTest.java
@@ -262,7 +262,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
cfg.setDefaultMode(mode);
if (mode != PRIMARY)
- cfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem(secondaryFileSystemUriPath(),
+ cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(secondaryFileSystemUriPath(),
secondaryFileSystemConfigPath()));
cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
@@ -327,7 +327,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
long used = 0, max = 0;
for (int i = 0; i < 4; i++) {
- IgniteFs igfs = grid(i).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(i).fileSystem("igfs");
IgfsMetrics metrics = igfs.metrics();
@@ -1298,7 +1298,7 @@ public abstract class HadoopIgfs20FileSystemAbstractSelfTest extends IgfsCommonA
out.write(new byte[1024 * 1024]);
}
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
IgfsPath filePath = new IgfsPath("/someFile");
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
index be7bb6d..a86b335 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
@@ -24,6 +24,7 @@ import org.apache.ignite.*;
import org.apache.ignite.cache.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.hadoop.fs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.processors.igfs.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -123,7 +124,7 @@ public abstract class HadoopIgfsDualAbstractSelfTest extends IgfsCommonAbstractT
* @throws Exception If failed.
*/
protected Ignite startGridWithIgfs(String gridName, String igfsName, IgfsMode mode,
- @Nullable Igfs secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
+ @Nullable IgfsSecondaryFileSystem secondaryFs, @Nullable Map<String, String> restCfg) throws Exception {
IgfsConfiguration igfsCfg = new IgfsConfiguration();
igfsCfg.setDataCacheName("dataCache");
@@ -183,7 +184,7 @@ public abstract class HadoopIgfsDualAbstractSelfTest extends IgfsCommonAbstractT
Ignite igniteSecondary = startGridWithIgfs("grid-secondary", "igfs-secondary", PRIMARY, null, SECONDARY_REST_CFG);
- Igfs hadoopFs = new IgniteHadoopSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG);
+ IgfsSecondaryFileSystem hadoopFs = new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG);
Ignite ignite = startGridWithIgfs("grid", "igfs", mode, hadoopFs, PRIMARY_REST_CFG);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
index 1688dec..de5aea1 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgfsEventsTestSuite.java
@@ -110,13 +110,13 @@ public class IgfsEventsTestSuite extends TestSuite {
*/
public abstract static class PrimarySecondaryTest extends IgfsEventsAbstractSelfTest {
/** Secondary file system. */
- private static IgniteFs igfsSec;
+ private static IgniteFileSystem igfsSec;
/** {@inheritDoc} */
@Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
- igfsCfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem(
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
"igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/",
"modules/core/src/test/config/hadoop/core-site-secondary.xml"));
@@ -167,7 +167,7 @@ public class IgfsEventsTestSuite extends TestSuite {
* @return Secondary file system handle.
* @throws Exception If failed.
*/
- @Nullable private IgniteFs startSecondary() throws Exception {
+ @Nullable private IgniteFileSystem startSecondary() throws Exception {
IgniteConfiguration cfg = getConfiguration("grid-secondary", getSecondaryIgfsConfiguration());
cfg.setLocalHost("127.0.0.1");
@@ -215,7 +215,7 @@ public class IgfsEventsTestSuite extends TestSuite {
@Override protected IgfsConfiguration getIgfsConfiguration() throws IgniteCheckedException {
IgfsConfiguration igfsCfg = super.getIgfsConfiguration();
- igfsCfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem(
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
"igfs://igfs-secondary:grid-secondary@127.0.0.1:11500/",
"modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
index 05a9e62..ffafd90 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemAbstractSelfTest.java
@@ -329,7 +329,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
cfg.setDefaultMode(mode);
if (mode != PRIMARY)
- cfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG_PATH));
+ cfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(SECONDARY_URI, SECONDARY_CFG_PATH));
cfg.setIpcEndpointConfiguration(primaryIpcEndpointConfiguration(gridName));
@@ -1537,7 +1537,7 @@ public abstract class IgniteHadoopFileSystemAbstractSelfTest extends IgfsCommonA
out.write(new byte[1024 * 1024]);
}
- IgniteFs igfs = grid(0).fileSystem("igfs");
+ IgniteFileSystem igfs = grid(0).fileSystem("igfs");
IgfsPath filePath = new IgfsPath("/someFile");
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
index 4632704..6208ce0 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemHandshakeSelfTest.java
@@ -175,7 +175,7 @@ public class IgniteHadoopFileSystemHandshakeSelfTest extends IgfsCommonAbstractT
private void startUp(boolean dfltGridName, boolean dfltIgfsName) throws Exception {
Ignite ignite = G.start(gridConfiguration(dfltGridName, dfltIgfsName));
- IgniteFs igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME);
+ IgniteFileSystem igfs = ignite.fileSystem(dfltIgfsName ? null : IGFS_NAME);
igfs.mkdirs(PATH);
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
index e672f4d..e99f7d6 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/IgniteHadoopFileSystemSecondaryModeSelfTest.java
@@ -97,7 +97,7 @@ public class IgniteHadoopFileSystemSecondaryModeSelfTest extends IgfsCommonAbstr
}});
igfsCfg.setManagementPort(-1);
- igfsCfg.setSecondaryFileSystem(new IgniteHadoopSecondaryFileSystem(
+ igfsCfg.setSecondaryFileSystem(new IgniteHadoopIgfsSecondaryFileSystem(
"igfs://igfs-secondary:igfs-grid-secondary@127.0.0.1:11500/",
"modules/core/src/test/config/hadoop/core-site-loopback-secondary.xml"));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
index 77e20fe..f43419c 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
@@ -71,7 +71,7 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes
private static final MockIgnite GRID = new MockIgnite();
/** Mocked IGFS. */
- private static final IgniteFs IGFS = new MockIgfs();
+ private static final IgniteFileSystem IGFS = new MockIgfs();
/** Planner. */
private static final HadoopMapReducePlanner PLANNER = new HadoopDefaultMapReducePlanner();
@@ -903,7 +903,7 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes
}
/** {@inheritDoc} */
- @Override public IgniteFs withAsync() {
+ @Override public IgniteFileSystem withAsync() {
return null;
}
@@ -929,7 +929,7 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes
}
/** {@inheritDoc} */
- @Override public IgniteFs igfsx(String name) {
+ @Override public IgniteFileSystem igfsx(String name) {
assert F.eq("igfs", name);
return IGFS;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
index e321191..6b711eb 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopTaskExecutionSelfTest.java
@@ -246,7 +246,7 @@ public class HadoopTaskExecutionSelfTest extends HadoopAbstractSelfTest {
* @throws Exception If failed.
*/
private void prepareFile(String fileName, int lineCnt) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName);
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName);
try (OutputStream os = igfs.create(new IgfsPath(fileName), true)) {
PrintWriter w = new PrintWriter(new OutputStreamWriter(os));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
index 837db37..59ac445 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/taskexecutor/external/HadoopExternalTaskExecutionSelfTest.java
@@ -148,7 +148,7 @@ public class HadoopExternalTaskExecutionSelfTest extends HadoopAbstractSelfTest
* @throws Exception If failed.
*/
private void prepareTestFile(String filePath) throws Exception {
- IgniteFs igfs = grid(0).fileSystem(igfsName);
+ IgniteFileSystem igfs = grid(0).fileSystem(igfsName);
try (IgfsOutputStream out = igfs.create(new IgfsPath(filePath), true)) {
PrintWriter wr = new PrintWriter(new OutputStreamWriter(out));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
----------------------------------------------------------------------
diff --git a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
index 31aa9e5..0f0a93f 100644
--- a/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
+++ b/modules/spring/src/main/java/org/apache/ignite/IgniteSpringBean.java
@@ -254,14 +254,14 @@ public class IgniteSpringBean implements Ignite, DisposableBean, InitializingBea
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
assert g != null;
return g.fileSystem(name);
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
assert g != null;
return g.fileSystems();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 79fe6c8..49f01cf 100644
--- a/pom.xml
+++ b/pom.xml
@@ -737,7 +737,7 @@
</group>
<group>
<title>Hadoop Accelerator APIs</title>
- <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs;org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce:org.apache.ignite.igfs;org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records</packages>
+ <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs:org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce:org.apache.ignite.igfs:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.igfs.mapreduce.secondary</packages>
</group>
<group>
<title>Streaming APIs</title>
@@ -932,7 +932,7 @@
</group>
<group>
<title>Hadoop Accelerator APIs</title>
- <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs;org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce:org.apache.ignite.igfs;org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records</packages>
+ <packages>org.apache.ignite.hadoop:org.apache.ignite.hadoop.fs:org.apache.ignite.hadoop.fs.v1:org.apache.ignite.hadoop.fs.v2:org.apache.ignite.hadoop.mapreduce:org.apache.ignite.igfs:org.apache.ignite.igfs.mapreduce:org.apache.ignite.igfs.mapreduce.records:org.apache.ignite.igfs.mapreduce.secondary</packages>
</group>
<group>
<title>Streaming APIs</title>
[5/6] incubator-ignite git commit: # IGNITE-386: WIP.
Posted by vo...@apache.org.
# IGNITE-386: WIP.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/2cbc7ea5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/2cbc7ea5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/2cbc7ea5
Branch: refs/heads/ignite-386
Commit: 2cbc7ea5a16318dfd1a3bf2c946c4fa646eef5e9
Parents: 7ef9198
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Mar 3 18:02:28 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Mar 3 18:02:28 2015 +0300
----------------------------------------------------------------------
.../org/apache/ignite/igfs/IgfsInputStream.java | 2 +-
.../igfs/secondary/IgfsSecondaryFileSystem.java | 2 +-
...fsSecondaryFileSystemPositionedReadable.java | 38 +++++++
.../IgfsSecondaryFileSystemReader.java | 38 -------
.../processors/igfs/IgfsDataManager.java | 2 +-
.../internal/processors/igfs/IgfsImpl.java | 2 +-
.../processors/igfs/IgfsInputStreamImpl.java | 4 +-
.../processors/igfs/IgfsMetaManager.java | 2 +-
.../IgfsSecondaryInputStreamDescriptor.java | 6 +-
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 4 +-
...fsSecondaryFileSystemPositionedReadable.java | 104 +++++++++++++++++++
.../HadoopIgfsSecondaryFileSystemReader.java | 104 -------------------
12 files changed, 154 insertions(+), 154 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
index 746aa6f..79d69a8 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
@@ -25,7 +25,7 @@ import java.io.*;
* {@code IGFS} input stream to read data from the file system.
* It provides several additional methods for asynchronous access.
*/
-public abstract class IgfsInputStream extends InputStream implements IgfsSecondaryFileSystemReader {
+public abstract class IgfsInputStream extends InputStream implements IgfsSecondaryFileSystemPositionedReadable {
/**
* Gets file length during file open.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index f306d67..abefd74 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -141,7 +141,7 @@ public interface IgfsSecondaryFileSystem {
* @throws IgniteException In case of error.
* @throws IgfsFileNotFoundException If path doesn't exist.
*/
- public IgfsSecondaryFileSystemReader open(IgfsPath path, int bufSize) throws IgniteException;
+ public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) throws IgniteException;
/**
* Creates a file and opens it for writing.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
new file mode 100644
index 0000000..3b43c2a
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemPositionedReadable.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.secondary;
+
+import java.io.*;
+
+/**
+ * The simplest data input interface to read from secondary file system in dual modes.
+ */
+public interface IgfsSecondaryFileSystemPositionedReadable extends Closeable {
+ /**
+ * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
+ * read.
+ *
+ * @param pos Position in the input stream to seek.
+ * @param buf Buffer into which data is read.
+ * @param off Offset in the buffer from which stream data should be written.
+ * @param len The number of bytes to read.
+ * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
+ * @throws IOException In case of any exception.
+ */
+ public int read(long pos, byte[] buf, int off, int len) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
deleted file mode 100644
index 02a135e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs.secondary;
-
-import java.io.*;
-
-/**
- * The simplest data input interface to read from secondary file system in dual modes.
- */
-public interface IgfsSecondaryFileSystemReader extends Closeable {
- /**
- * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
- * read.
- *
- * @param pos Position in the input stream to seek.
- * @param buf Buffer into which data is read.
- * @param off Offset in the buffer from which stream data should be written.
- * @param len The number of bytes to read.
- * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
- * @throws IOException In case of any exception.
- */
- public int read(long pos, byte[] buf, int off, int len) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index 964a5c2..333c5ca 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -383,7 +383,7 @@ public class IgfsDataManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
@Nullable public IgniteInternalFuture<byte[]> dataBlock(final IgfsFileInfo fileInfo, final IgfsPath path,
- final long blockIdx, @Nullable final IgfsSecondaryFileSystemReader secReader)
+ final long blockIdx, @Nullable final IgfsSecondaryFileSystemPositionedReadable secReader)
throws IgniteCheckedException {
//assert validTxState(any); // Allow this method call for any transaction state.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index 5d11fe7..c3c81f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -2034,7 +2034,7 @@ public final class IgfsImpl implements IgfsEx {
* @param metrics Metrics.
*/
IgfsEventAwareInputStream(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo,
- int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemReader secReader,
+ int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemPositionedReadable secReader,
IgfsLocalMetrics metrics) {
super(igfsCtx, path, fileInfo, prefetchBlocks, seqReadsBeforePrefetch, secReader, metrics);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
index c38103d..5afa523 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
@@ -46,7 +46,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
/** Secondary file system reader. */
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private final IgfsSecondaryFileSystemReader secReader;
+ private final IgfsSecondaryFileSystemPositionedReadable secReader;
/** Logger. */
private IgniteLogger log;
@@ -111,7 +111,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
* @param metrics Local IGFS metrics.
*/
IgfsInputStreamImpl(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo, int prefetchBlocks,
- int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemReader secReader, IgfsLocalMetrics metrics) {
+ int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemPositionedReadable secReader, IgfsLocalMetrics metrics) {
assert igfsCtx != null;
assert path != null;
assert fileInfo != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index ea95875..f2fc9f9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -1784,7 +1784,7 @@ public class IgfsMetaManager extends IgfsManager {
if (remainder > 0) {
int blockIdx = (int)(len / blockSize);
- IgfsSecondaryFileSystemReader reader = fs.open(path, bufSize);
+ IgfsSecondaryFileSystemPositionedReadable reader = fs.open(path, bufSize);
try {
igfsCtx.data().dataBlock(info, path, blockIdx, reader).get();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
index b9728cd..e9ba6f4 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
@@ -27,7 +27,7 @@ public class IgfsSecondaryInputStreamDescriptor {
private final IgfsFileInfo info;
/** Secondary file system input stream wrapper. */
- private final IgfsSecondaryFileSystemReader secReader;
+ private final IgfsSecondaryFileSystemPositionedReadable secReader;
/**
* Constructor.
@@ -35,7 +35,7 @@ public class IgfsSecondaryInputStreamDescriptor {
* @param info File info in the primary file system.
* @param secReader Secondary file system reader.
*/
- IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsSecondaryFileSystemReader secReader) {
+ IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsSecondaryFileSystemPositionedReadable secReader) {
assert info != null;
assert secReader != null;
@@ -53,7 +53,7 @@ public class IgfsSecondaryInputStreamDescriptor {
/**
* @return Secondary file system reader.
*/
- IgfsSecondaryFileSystemReader reader() {
+ IgfsSecondaryFileSystemPositionedReadable reader() {
return secReader;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 3e5f32f..dfdbe91 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -275,8 +275,8 @@ public class IgniteHadoopIgfsSecondaryFileSystem implements IgfsSecondaryFileSys
}
/** {@inheritDoc} */
- @Override public IgfsSecondaryFileSystemReader open(IgfsPath path, int bufSize) {
- return new HadoopIgfsSecondaryFileSystemReader(fileSys, convert(path), bufSize);
+ @Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize) {
+ return new HadoopIgfsSecondaryFileSystemPositionedReadable(fileSys, convert(path), bufSize);
}
/** {@inheritDoc} */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
new file mode 100644
index 0000000..e921fd3
--- /dev/null
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemPositionedReadable.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.hadoop.igfs;
+
+import org.apache.hadoop.fs.*;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+
+import java.io.*;
+
+/**
+ * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
+ * requested.
+ * <p>
+ * The class is expected to be used only from synchronized context and therefore is not tread-safe.
+ */
+public class HadoopIgfsSecondaryFileSystemPositionedReadable implements IgfsSecondaryFileSystemPositionedReadable {
+ /** Secondary file system. */
+ private final FileSystem fs;
+
+ /** Path to the file to open. */
+ private final Path path;
+
+ /** Buffer size. */
+ private final int bufSize;
+
+ /** Actual input stream. */
+ private FSDataInputStream in;
+
+ /** Cached error occurred during output stream open. */
+ private IOException err;
+
+ /** Flag indicating that the stream was already opened. */
+ private boolean opened;
+
+ /**
+ * Constructor.
+ *
+ * @param fs Secondary file system.
+ * @param path Path to the file to open.
+ * @param bufSize Buffer size.
+ */
+ public HadoopIgfsSecondaryFileSystemPositionedReadable(FileSystem fs, Path path, int bufSize) {
+ assert fs != null;
+ assert path != null;
+
+ this.fs = fs;
+ this.path = path;
+ this.bufSize = bufSize;
+ }
+
+ /** Get input stream. */
+ private PositionedReadable in() throws IOException {
+ if (opened) {
+ if (err != null)
+ throw err;
+ }
+ else {
+ opened = true;
+
+ try {
+ in = fs.open(path, bufSize);
+
+ if (in == null)
+ throw new IOException("Failed to open input stream (file system returned null): " + path);
+ }
+ catch (IOException e) {
+ err = e;
+
+ throw err;
+ }
+ }
+
+ return in;
+ }
+
+ /**
+ * Close wrapped input stream in case it was previously opened.
+ */
+ @Override public void close() {
+ U.closeQuiet(in);
+ }
+
+ /** {@inheritDoc} */
+ @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
+ return in().read(pos, buf, off, len);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/2cbc7ea5/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
deleted file mode 100644
index 15d5419..0000000
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsSecondaryFileSystemReader.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.hadoop.igfs;
-
-import org.apache.hadoop.fs.*;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.ignite.igfs.secondary.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-
-import java.io.*;
-
-/**
- * Secondary file system input stream wrapper which actually opens input stream only in case it is explicitly
- * requested.
- * <p>
- * The class is expected to be used only from synchronized context and therefore is not tread-safe.
- */
-public class HadoopIgfsSecondaryFileSystemReader implements IgfsSecondaryFileSystemReader {
- /** Secondary file system. */
- private final FileSystem fs;
-
- /** Path to the file to open. */
- private final Path path;
-
- /** Buffer size. */
- private final int bufSize;
-
- /** Actual input stream. */
- private FSDataInputStream in;
-
- /** Cached error occurred during output stream open. */
- private IOException err;
-
- /** Flag indicating that the stream was already opened. */
- private boolean opened;
-
- /**
- * Constructor.
- *
- * @param fs Secondary file system.
- * @param path Path to the file to open.
- * @param bufSize Buffer size.
- */
- public HadoopIgfsSecondaryFileSystemReader(FileSystem fs, Path path, int bufSize) {
- assert fs != null;
- assert path != null;
-
- this.fs = fs;
- this.path = path;
- this.bufSize = bufSize;
- }
-
- /** Get input stream. */
- private PositionedReadable in() throws IOException {
- if (opened) {
- if (err != null)
- throw err;
- }
- else {
- opened = true;
-
- try {
- in = fs.open(path, bufSize);
-
- if (in == null)
- throw new IOException("Failed to open input stream (file system returned null): " + path);
- }
- catch (IOException e) {
- err = e;
-
- throw err;
- }
- }
-
- return in;
- }
-
- /**
- * Close wrapped input stream in case it was previously opened.
- */
- @Override public void close() {
- U.closeQuiet(in);
- }
-
- /** {@inheritDoc} */
- @Override public int read(long pos, byte[] buf, int off, int len) throws IOException {
- return in().read(pos, buf, off, len);
- }
-}
[2/6] incubator-ignite git commit: # IGNITE-386: WIP.
Posted by vo...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
index e960422..964a5c2 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsDataManager.java
@@ -24,6 +24,7 @@ import org.apache.ignite.cluster.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.cluster.*;
import org.apache.ignite.internal.managers.communication.*;
@@ -382,7 +383,7 @@ public class IgfsDataManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
@Nullable public IgniteInternalFuture<byte[]> dataBlock(final IgfsFileInfo fileInfo, final IgfsPath path,
- final long blockIdx, @Nullable final IgfsReader secReader)
+ final long blockIdx, @Nullable final IgfsSecondaryFileSystemReader secReader)
throws IgniteCheckedException {
//assert validTxState(any); // Allow this method call for any transaction state.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index e067e78..f5579b9 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -26,9 +26,9 @@ import org.jetbrains.annotations.*;
import java.net.*;
/**
- * Internal API extension for {@link org.apache.ignite.IgniteFs}.
+ * Internal API extension for {@link org.apache.ignite.IgniteFileSystem}.
*/
-public interface IgfsEx extends IgniteFs {
+public interface IgfsEx extends IgniteFileSystem {
/**
* Stops IGFS cleaning all used resources.
*/
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index eff987e..5d11fe7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -26,6 +26,7 @@ import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.managers.communication.*;
import org.apache.ignite.internal.managers.eventstorage.*;
@@ -89,7 +90,7 @@ public final class IgfsImpl implements IgfsEx {
private final IgfsModeResolver modeRslvr;
/** Connection to the secondary file system. */
- private Igfs secondaryFs;
+ private IgfsSecondaryFileSystem secondaryFs;
/** Busy lock. */
private final GridSpinBusyLock busyLock = new GridSpinBusyLock();
@@ -1898,7 +1899,7 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
- @Override public IgniteFs withAsync() {
+ @Override public IgniteFileSystem withAsync() {
return new IgfsAsyncImpl(this);
}
@@ -2033,7 +2034,7 @@ public final class IgfsImpl implements IgfsEx {
* @param metrics Metrics.
*/
IgfsEventAwareInputStream(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo,
- int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsReader secReader,
+ int prefetchBlocks, int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemReader secReader,
IgfsLocalMetrics metrics) {
super(igfsCtx, path, fileInfo, prefetchBlocks, seqReadsBeforePrefetch, secReader, metrics);
@@ -2083,7 +2084,7 @@ public final class IgfsImpl implements IgfsEx {
private Ignite g;
@Nullable @Override public IgniteBiTuple<Long, Long> execute() {
- IgniteFs igfs = ((IgniteKernal)g).context().igfs().igfs(igfsName);
+ IgniteFileSystem igfs = ((IgniteKernal)g).context().igfs().igfs(igfsName);
if (igfs == null)
return F.t(0L, 0L);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
index 92b4383..c38103d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamImpl.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -45,7 +46,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
/** Secondary file system reader. */
@SuppressWarnings("FieldAccessedSynchronizedAndUnsynchronized")
- private final IgfsReader secReader;
+ private final IgfsSecondaryFileSystemReader secReader;
/** Logger. */
private IgniteLogger log;
@@ -110,7 +111,7 @@ public class IgfsInputStreamImpl extends IgfsInputStreamAdapter {
* @param metrics Local IGFS metrics.
*/
IgfsInputStreamImpl(IgfsContext igfsCtx, IgfsPath path, IgfsFileInfo fileInfo, int prefetchBlocks,
- int seqReadsBeforePrefetch, @Nullable IgfsReader secReader, IgfsLocalMetrics metrics) {
+ int seqReadsBeforePrefetch, @Nullable IgfsSecondaryFileSystemReader secReader, IgfsLocalMetrics metrics) {
assert igfsCtx != null;
assert path != null;
assert fileInfo != null;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
index f6b17bf..fa90e21 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsJobImpl.java
@@ -79,7 +79,7 @@ public class IgfsJobImpl implements ComputeJob, GridInternalWrapper<IgfsJob> {
/** {@inheritDoc} */
@Override public Object execute() {
- IgniteFs fs = ignite.fileSystem(igfsName);
+ IgniteFileSystem fs = ignite.fileSystem(igfsName);
try (IgfsInputStream in = fs.open(path)) {
IgfsFileRange split = new IgfsFileRange(path, start, len);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
index 9d1795a..ea95875 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsMetaManager.java
@@ -23,6 +23,7 @@ import org.apache.ignite.cluster.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.events.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.managers.eventstorage.*;
import org.apache.ignite.internal.processors.cache.*;
@@ -1588,7 +1589,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Output stream descriptor.
* @throws IgniteCheckedException If file creation failed.
*/
- public IgfsSecondaryOutputStreamDescriptor createDual(final Igfs fs,
+ public IgfsSecondaryOutputStreamDescriptor createDual(final IgfsSecondaryFileSystem fs,
final IgfsPath path,
final boolean simpleCreate,
@Nullable final Map<String, String> props,
@@ -1752,7 +1753,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Output stream descriptor.
* @throws IgniteCheckedException If output stream open for append has failed.
*/
- public IgfsSecondaryOutputStreamDescriptor appendDual(final Igfs fs, final IgfsPath path,
+ public IgfsSecondaryOutputStreamDescriptor appendDual(final IgfsSecondaryFileSystem fs, final IgfsPath path,
final int bufSize) throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -1783,7 +1784,7 @@ public class IgfsMetaManager extends IgfsManager {
if (remainder > 0) {
int blockIdx = (int)(len / blockSize);
- IgfsReader reader = fs.open(path, bufSize);
+ IgfsSecondaryFileSystemReader reader = fs.open(path, bufSize);
try {
igfsCtx.data().dataBlock(info, path, blockIdx, reader).get();
@@ -1832,7 +1833,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Input stream descriptor.
* @throws IgniteCheckedException If input stream open has failed.
*/
- public IgfsSecondaryInputStreamDescriptor openDual(final Igfs fs, final IgfsPath path,
+ public IgfsSecondaryInputStreamDescriptor openDual(final IgfsSecondaryFileSystem fs, final IgfsPath path,
final int bufSize)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
@@ -1893,7 +1894,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return File info or {@code null} if file not found.
* @throws IgniteCheckedException If sync task failed.
*/
- @Nullable public IgfsFileInfo synchronizeFileDual(final Igfs fs, final IgfsPath path)
+ @Nullable public IgfsFileInfo synchronizeFileDual(final IgfsSecondaryFileSystem fs, final IgfsPath path)
throws IgniteCheckedException {
assert fs != null;
assert path != null;
@@ -1941,7 +1942,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return {@code True} in case rename was successful.
* @throws IgniteCheckedException If directory creation failed.
*/
- public boolean mkdirsDual(final Igfs fs, final IgfsPath path, final Map<String, String> props)
+ public boolean mkdirsDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final Map<String, String> props)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2025,7 +2026,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Operation result.
* @throws IgniteCheckedException If failed.
*/
- public boolean renameDual(final Igfs fs, final IgfsPath src, final IgfsPath dest) throws
+ public boolean renameDual(final IgfsSecondaryFileSystem fs, final IgfsPath src, final IgfsPath dest) throws
IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2124,7 +2125,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Operation result.
* @throws IgniteCheckedException If delete failed.
*/
- public boolean deleteDual(final Igfs fs, final IgfsPath path, final boolean recursive)
+ public boolean deleteDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final boolean recursive)
throws IgniteCheckedException {
if (busyLock.enterBusy()) {
try {
@@ -2190,7 +2191,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return Update file info.
* @throws IgniteCheckedException If update failed.
*/
- public IgfsFileInfo updateDual(final Igfs fs, final IgfsPath path, final Map<String, String> props)
+ public IgfsFileInfo updateDual(final IgfsSecondaryFileSystem fs, final IgfsPath path, final Map<String, String> props)
throws IgniteCheckedException {
assert fs != null;
assert path != null;
@@ -2243,7 +2244,7 @@ public class IgfsMetaManager extends IgfsManager {
* @return File info of the end path.
* @throws IgniteCheckedException If failed.
*/
- private IgfsFileInfo synchronize(Igfs fs,
+ private IgfsFileInfo synchronize(IgfsSecondaryFileSystem fs,
IgfsPath startPath,
IgfsFileInfo startPathInfo,
IgfsPath endPath,
@@ -2328,7 +2329,7 @@ public class IgfsMetaManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
private <T> T synchronizeAndExecute(SynchronizationTask<T> task,
- Igfs fs,
+ IgfsSecondaryFileSystem fs,
boolean strict,
IgfsPath... paths)
throws IgniteCheckedException
@@ -2349,7 +2350,7 @@ public class IgfsMetaManager extends IgfsManager {
* @throws IgniteCheckedException If failed.
*/
private <T> T synchronizeAndExecute(SynchronizationTask<T> task,
- Igfs fs,
+ IgfsSecondaryFileSystem fs,
boolean strict,
@Nullable Collection<IgniteUuid> extraLockIds,
IgfsPath... paths)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
index a299fc4..41dcc31 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsNoopProcessor.java
@@ -49,12 +49,12 @@ public class IgfsNoopProcessor extends IgfsProcessorAdapter {
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> igfss() {
+ @Override public Collection<IgniteFileSystem> igfss() {
return Collections.emptyList();
}
/** {@inheritDoc} */
- @Nullable @Override public IgniteFs igfs(@Nullable String name) {
+ @Nullable @Override public IgniteFileSystem igfs(@Nullable String name) {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
index 5941579..49e2f68 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
@@ -405,7 +405,7 @@ class IgfsOutputStreamImpl extends IgfsOutputStreamAdapter {
if (!igfsCtx.configuration().isFragmentizerEnabled())
return null;
- if (!Boolean.parseBoolean(fileInfo.properties().get(IgniteFs.PROP_PREFER_LOCAL_WRITES)))
+ if (!Boolean.parseBoolean(fileInfo.properties().get(IgniteFileSystem.PROP_PREFER_LOCAL_WRITES)))
return null;
int blockSize = fileInfo.blockSize();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
index 847cd50..8646aff 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessor.java
@@ -51,8 +51,8 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
private static final String NULL_NAME = UUID.randomUUID().toString();
/** Converts context to IGFS. */
- private static final IgniteClosure<IgfsContext,IgniteFs> CTX_TO_IGFS = new C1<IgfsContext, IgniteFs>() {
- @Override public IgniteFs apply(IgfsContext igfsCtx) {
+ private static final IgniteClosure<IgfsContext,IgniteFileSystem> CTX_TO_IGFS = new C1<IgfsContext, IgniteFileSystem>() {
+ @Override public IgniteFileSystem apply(IgfsContext igfsCtx) {
return igfsCtx.igfs();
}
};
@@ -167,12 +167,12 @@ public class IgfsProcessor extends IgfsProcessorAdapter {
/** {@inheritDoc} */
@SuppressWarnings("unchecked")
- @Override public Collection<IgniteFs> igfss() {
+ @Override public Collection<IgniteFileSystem> igfss() {
return F.viewReadOnly(igfsCache.values(), CTX_TO_IGFS);
}
/** {@inheritDoc} */
- @Override @Nullable public IgniteFs igfs(@Nullable String name) {
+ @Override @Nullable public IgniteFileSystem igfs(@Nullable String name) {
IgfsContext igfsCtx = igfsCache.get(maskName(name));
return igfsCtx == null ? null : igfsCtx.igfs();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
index b695104..ee38ab3 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsProcessorAdapter.java
@@ -46,7 +46,7 @@ public abstract class IgfsProcessorAdapter extends GridProcessorAdapter {
*
* @return Collection of IGFS instances.
*/
- public abstract Collection<IgniteFs> igfss();
+ public abstract Collection<IgniteFileSystem> igfss();
/**
* Gets IGFS instance.
@@ -54,7 +54,7 @@ public abstract class IgfsProcessorAdapter extends GridProcessorAdapter {
* @param name (Nullable) IGFS name.
* @return IGFS instance.
*/
- @Nullable public abstract IgniteFs igfs(@Nullable String name);
+ @Nullable public abstract IgniteFileSystem igfs(@Nullable String name);
/**
* Gets server endpoints for particular IGFS.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
index 6e48103..b9728cd 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryInputStreamDescriptor.java
@@ -17,7 +17,7 @@
package org.apache.ignite.internal.processors.igfs;
-import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
/**
* Descriptor of an input stream opened to the secondary file system.
@@ -27,7 +27,7 @@ public class IgfsSecondaryInputStreamDescriptor {
private final IgfsFileInfo info;
/** Secondary file system input stream wrapper. */
- private final IgfsReader secReader;
+ private final IgfsSecondaryFileSystemReader secReader;
/**
* Constructor.
@@ -35,7 +35,7 @@ public class IgfsSecondaryInputStreamDescriptor {
* @param info File info in the primary file system.
* @param secReader Secondary file system reader.
*/
- IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsReader secReader) {
+ IgfsSecondaryInputStreamDescriptor(IgfsFileInfo info, IgfsSecondaryFileSystemReader secReader) {
assert info != null;
assert secReader != null;
@@ -53,7 +53,7 @@ public class IgfsSecondaryInputStreamDescriptor {
/**
* @return Secondary file system reader.
*/
- IgfsReader reader() {
+ IgfsSecondaryFileSystemReader reader() {
return secReader;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
index 2258d4d..edcde6a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfs.java
@@ -24,7 +24,7 @@ import org.apache.ignite.internal.util.typedef.internal.*;
import java.io.*;
/**
- * Data transfer object for {@link org.apache.ignite.IgniteFs}.
+ * Data transfer object for {@link org.apache.ignite.IgniteFileSystem}.
*/
public class VisorIgfs implements Serializable {
/** */
@@ -66,7 +66,7 @@ public class VisorIgfs implements Serializable {
* @param igfs Source IGFS.
* @return Data transfer object for given IGFS.
*/
- public static VisorIgfs from(IgniteFs igfs) {
+ public static VisorIgfs from(IgniteFileSystem igfs) {
assert igfs != null;
return new VisorIgfs(
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
index f63e669..814fc50 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/igfs/VisorIgfsProfilerClearTask.java
@@ -59,7 +59,7 @@ public class VisorIgfsProfilerClearTask extends VisorOneNodeTask<String, IgniteB
int notDeleted = 0;
try {
- IgniteFs igfs = ignite.fileSystem(arg);
+ IgniteFileSystem igfs = ignite.fileSystem(arg);
Path logsDir = resolveIgfsProfilerLogsDir(igfs);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
index 056ac7f..fbb751a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorIgfsConfiguration.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.visor.node;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
@@ -137,7 +138,7 @@ public class VisorIgfsConfiguration implements Serializable {
cfg.perNodeBatchSize = igfs.getPerNodeBatchSize();
cfg.perNodeParallelBatchCnt = igfs.getPerNodeParallelBatchCount();
- Igfs secFs = igfs.getSecondaryFileSystem();
+ IgfsSecondaryFileSystem secFs = igfs.getSecondaryFileSystem();
if (secFs != null) {
Map<String, String> props = secFs.properties();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
index 43a987c..b86ea5a 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/node/VisorNodeDataCollectorJob.java
@@ -151,7 +151,7 @@ public class VisorNodeDataCollectorJob extends VisorJob<VisorNodeDataCollectorTa
try {
IgfsProcessorAdapter igfsProc = ((IgniteKernal)ignite).context().igfs();
- for (IgniteFs igfs : igfsProc.igfss()) {
+ for (IgniteFileSystem igfs : igfsProc.igfss()) {
long start0 = U.currentTimeMillis();
try {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
index b03674f..31acae7 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/util/VisorTaskUtils.java
@@ -592,7 +592,7 @@ public class VisorTaskUtils {
* @return {@link Path} to log dir or {@code null} if not found.
* @throws IgniteCheckedException if failed to resolve.
*/
- public static Path resolveIgfsProfilerLogsDir(IgniteFs igfs) throws IgniteCheckedException {
+ public static Path resolveIgfsProfilerLogsDir(IgniteFileSystem igfs) throws IgniteCheckedException {
String logsDir;
if (igfs instanceof IgfsEx)
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite_new/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite_new/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite_new/IgniteFileSystem.java
deleted file mode 100644
index 5042b93..0000000
--- a/modules/core/src/main/java/org/apache/ignite_new/IgniteFileSystem.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite_new;
-
-import org.apache.ignite.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
- * <ul>
- * <li>list directories or get information for a single path</li>
- * <li>create/move/delete files or directories</li>
- * <li>write/read data streams into/from files</li>
- * </ul>
- * The data of each file is split on separate data blocks and stored in the cache.
- * You can access file's data with a standard Java streaming API. Moreover, for each part
- * of the file you can calculate an affinity and process file's content on corresponding
- * nodes to escape unnecessary networking.
- * <p/>
- * This API is fully thread-safe and you can use it from several threads.
- * <h1 class="header">IGFS Configuration</h1>
- * The simplest way to run a Ignite node with configured file system is to pass
- * special configuration file included in Ignite distribution to {@code ignite.sh} or
- * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml}
- * <p>
- * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for
- * caching data, while client node is responsible for basic file system operations and accessing
- * data nodes remotely. When used as Hadoop file system, clients nodes usually started together
- * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
- * started together with Hadoop {@code task-tracker} processes.
- * <h1 class="header">Integration With Hadoop</h1>
- * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by
- * plugging in as {@code Hadoop FileSystem}. Refer to
- * {@code org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem} or
- * {@code org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem} for more information.
- * <p>
- * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- */
-public interface IgniteFileSystem extends IgniteAsyncSupport {
- /** IGFS scheme name. */
- public static final String IGFS_SCHEME = "igfs";
-
- /** File property: prefer writes to local node. */
- public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
-
- /**
- * Gets IGFS name.
- *
- * @return IGFS name, or {@code null} for default file system.
- */
- public String name();
-
- /**
- * Gets IGFS configuration.
- *
- * @return IGFS configuration.
- */
- public IgfsConfiguration configuration();
-
- /**
- * Gets summary (total number of files, total number of directories and total length)
- * for a given path.
- *
- * @param path Path to get information for.
- * @return Summary object.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found.
- * @throws IgniteException If failed.
- */
- public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be
- * stored on node where {@code affKey} resides.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
- @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param create Create file if it doesn't exist yet.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Sets last access time and last modification time for a given path. If argument is {@code null},
- * corresponding time will not be changed.
- *
- * @param path Path to update.
- * @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
- * @param modificationTime Optional last modification time to set. Value {@code -1} does not update
- * modification time.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found.
- * @throws IgniteException If error occurred.
- */
- public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
-
- /**
- * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks
- * are stored.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
-
- /**
- * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and
- * particular block location length is greater than this value, block locations will be split into smaller
- * chunks.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @param maxLen Maximum length of a single returned block location length.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
- throws IgniteException;
-
- /**
- * Gets metrics snapshot for this file system.
- *
- * @return Metrics.
- * @throws IgniteException In case of error.
- */
- public IgfsMetrics metrics() throws IgniteException;
-
- /**
- * Resets metrics for this file system.
- *
- * @throws IgniteException In case of error.
- */
- public void resetMetrics() throws IgniteException;
-
- /**
- * Determines size of the file denoted by provided path. In case if path is a directory, then
- * total size of all containing entries will be calculated recursively.
- *
- * @param path File system path.
- * @return Total size.
- * @throws IgniteException In case of error.
- */
- public long size(IgfsPath path) throws IgniteException;
-
- /**
- * Formats the file system removing all existing entries from it.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @throws IgniteException In case format has failed.
- */
- @IgniteAsyncSupported
- public void format() throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive
- * IGFS blocks will be included.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg)
- throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Maximum range length.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
- long maxRangeLen, @Nullable T arg) throws IgniteException;
-
- /** {@inheritDoc} */
- @Override public IgniteFs withAsync();
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- /**
- * Checks if the specified path exists in the file system.
- *
- * @param path Path to check for existence in the file system.
- * @return {@code True} if such file exists, otherwise - {@code false}.
- * @throws IgniteException In case of error.
- */
- public boolean exists(IgfsPath path);
-
- /**
- * Updates file information for the specified path. Existent properties, not listed in the passed collection,
- * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
- * will be removed from the stored properties or ignored if they don't exist in the file info.
- * <p>
- * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
- * to the secondary file system:
- * <ul>
- * <li>{@code usrName} - file owner name;</li>
- * <li>{@code grpName} - file owner group;</li>
- * <li>{@code permission} - Unix-style string representing file permissions.</li>
- * </ul>
- *
- * @param path File path to set properties for.
- * @param props Properties to update.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
-
- /**
- * Renames/moves a file.
- * <p>
- * You are free to rename/move data files as you wish, but directories can be only renamed.
- * You cannot move the directory between different parent directories.
- * <p>
- * Examples:
- * <ul>
- * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
- * <li>"/work" => "/work-2012.bkp"</li>
- * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
- * </ul>
- *
- * @param src Source file path to rename.
- * @param dest Destination file path. If destination path is a directory, then source file will be placed
- * into destination directory with original name.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If source file doesn't exist.
- */
- public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
-
- /**
- * Deletes file.
- *
- * @param path File path to delete.
- * @param recursive Delete non-empty directories recursively.
- * @return {@code True} in case of success, {@code false} otherwise.
- * @throws IgniteException In case of error.
- */
- public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
-
- /**
- * Creates directories under specified path.
- *
- * @param path Path of directories chain to create.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path) throws IgniteException;
-
- /**
- * Creates directories under specified path with the specified properties.
- *
- * @param path Path of directories chain to create.
- * @param props Metadata properties to set on created directories.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Lists file paths under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
-
- /**
- * Lists files under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
-
- /**
- * Gets file information for the specified path.
- *
- * @param path Path to get information for.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile info(IgfsPath path) throws IgniteException;
-
- /**
- * Gets used space in bytes.
- *
- * @return Used space in bytes.
- * @throws IgniteException In case of error.
- */
- public long usedSpaceSize() throws IgniteException;
-
- /**
- * Gets the implementation specific properties of file system.
- *
- * @return Map of properties.
- */
- public Map<String,String> properties();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite_new/configuration/FileSystemConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite_new/configuration/FileSystemConfiguration.java b/modules/core/src/main/java/org/apache/ignite_new/configuration/FileSystemConfiguration.java
deleted file mode 100644
index ea838fa..0000000
--- a/modules/core/src/main/java/org/apache/ignite_new/configuration/FileSystemConfiguration.java
+++ /dev/null
@@ -1,809 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite_new.configuration;
-
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite_new.filesystem.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-/**
- * {@code IGFS} configuration. More than one file system can be configured within grid.
- * {@code IGFS} configuration is provided via
- * {@link org.apache.ignite.configuration.IgniteConfiguration#getIgfsConfiguration()}
- * method.
- * <p>
- * Refer to {@code config/hadoop/default-config.xml} or {@code config/hadoop/default-config-client.xml}
- * configuration files under Ignite installation to see sample {@code IGFS} configuration.
- */
-public class FileSystemConfiguration {
- /** Default file system user name. */
- public static final String DFLT_USER_NAME = System.getProperty("user.name", "anonymous");
-
- /** Default IPC port. */
- public static final int DFLT_IPC_PORT = 10500;
-
- /** Default fragmentizer throttling block length. */
- public static final long DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH = 16 * 1024 * 1024;
-
- /** Default fragmentizer throttling delay. */
- public static final long DFLT_FRAGMENTIZER_THROTTLING_DELAY = 200;
-
- /** Default fragmentizer concurrent files. */
- public static final int DFLT_FRAGMENTIZER_CONCURRENT_FILES = 0;
-
- /** Default fragmentizer local writes ratio. */
- public static final float DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO = 0.8f;
-
- /** Fragmentizer enabled property. */
- public static final boolean DFLT_FRAGMENTIZER_ENABLED = true;
-
- /** Default batch size for logging. */
- public static final int DFLT_IGFS_LOG_BATCH_SIZE = 100;
-
- /** Default {@code IGFS} log directory. */
- public static final String DFLT_IGFS_LOG_DIR = "work/igfs/log";
-
- /** Default per node buffer size. */
- public static final int DFLT_PER_NODE_BATCH_SIZE = 100;
-
- /** Default number of per node parallel operations. */
- public static final int DFLT_PER_NODE_PARALLEL_BATCH_CNT = 8;
-
- /** Default IGFS mode. */
- public static final IgfsMode DFLT_MODE = IgfsMode.DUAL_ASYNC;
-
- /** Default file's data block size (bytes). */
- public static final int DFLT_BLOCK_SIZE = 1 << 16;
-
- /** Default read/write buffers size (bytes). */
- public static final int DFLT_BUF_SIZE = 1 << 16;
-
- /** Default trash directory purge await timeout in case data cache oversize is detected. */
- public static final long DFLT_TRASH_PURGE_TIMEOUT = 1000;
-
- /** Default management port. */
- public static final int DFLT_MGMT_PORT = 11400;
-
- /** Default IPC endpoint enabled flag. */
- public static final boolean DFLT_IPC_ENDPOINT_ENABLED = true;
-
- /** IGFS instance name. */
- private String name;
-
- /** Cache name to store IGFS meta information. */
- private String metaCacheName;
-
- /** Cache name to store file's data blocks. */
- private String dataCacheName;
-
- /** File's data block size (bytes). */
- private int blockSize = DFLT_BLOCK_SIZE;
-
- /** The number of pre-fetched blocks if specific file's chunk is requested. */
- private int prefetchBlocks;
-
- /** Amount of sequential block reads before prefetch is triggered. */
- private int seqReadsBeforePrefetch;
-
- /** Read/write buffers size for stream operations (bytes). */
- private int bufSize = DFLT_BUF_SIZE;
-
- /** Per node buffer size. */
- private int perNodeBatchSize = DFLT_PER_NODE_BATCH_SIZE;
-
- /** Per node parallel operations. */
- private int perNodeParallelBatchCnt = DFLT_PER_NODE_PARALLEL_BATCH_CNT;
-
- /** IPC endpoint properties to publish IGFS over. */
- private Map<String, String> ipcEndpointCfg;
-
- /** IPC endpoint enabled flag. */
- private boolean ipcEndpointEnabled = DFLT_IPC_ENDPOINT_ENABLED;
-
- /** Management port. */
- private int mgmtPort = DFLT_MGMT_PORT;
-
- /** Secondary file system */
- private SecondaryFileSystem secondaryFs;
-
- /** IGFS mode. */
- private IgfsMode dfltMode = DFLT_MODE;
-
- /** Fragmentizer throttling block length. */
- private long fragmentizerThrottlingBlockLen = DFLT_FRAGMENTIZER_THROTTLING_BLOCK_LENGTH;
-
- /** Fragmentizer throttling delay. */
- private long fragmentizerThrottlingDelay = DFLT_FRAGMENTIZER_THROTTLING_DELAY;
-
- /** Fragmentizer concurrent files. */
- private int fragmentizerConcurrentFiles = DFLT_FRAGMENTIZER_CONCURRENT_FILES;
-
- /** Fragmentizer local writes ratio. */
- private float fragmentizerLocWritesRatio = DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO;
-
- /** Fragmentizer enabled flag. */
- private boolean fragmentizerEnabled = DFLT_FRAGMENTIZER_ENABLED;
-
- /** Path modes. */
- private Map<String, IgfsMode> pathModes;
-
- /** Maximum space. */
- private long maxSpace;
-
- /** Trash purge await timeout. */
- private long trashPurgeTimeout = DFLT_TRASH_PURGE_TIMEOUT;
-
- /** Dual mode PUT operations executor service. */
- private ExecutorService dualModePutExec;
-
- /** Dual mode PUT operations executor service shutdown flag. */
- private boolean dualModePutExecShutdown;
-
- /** Maximum amount of data in pending puts. */
- private long dualModeMaxPendingPutsSize;
-
- /** Maximum range length. */
- private long maxTaskRangeLen;
-
- /**
- * Constructs default configuration.
- */
- public FileSystemConfiguration() {
- // No-op.
- }
-
- /**
- * Constructs the copy of the configuration.
- *
- * @param cfg Configuration to copy.
- */
- public FileSystemConfiguration(FileSystemConfiguration cfg) {
- assert cfg != null;
-
- /*
- * Must preserve alphabetical order!
- */
- blockSize = cfg.getBlockSize();
- bufSize = cfg.getStreamBufferSize();
- dataCacheName = cfg.getDataCacheName();
- dfltMode = cfg.getDefaultMode();
- dualModeMaxPendingPutsSize = cfg.getDualModeMaxPendingPutsSize();
- dualModePutExec = cfg.getDualModePutExecutorService();
- dualModePutExecShutdown = cfg.getDualModePutExecutorServiceShutdown();
- fragmentizerConcurrentFiles = cfg.getFragmentizerConcurrentFiles();
- fragmentizerLocWritesRatio = cfg.getFragmentizerLocalWritesRatio();
- fragmentizerEnabled = cfg.isFragmentizerEnabled();
- fragmentizerThrottlingBlockLen = cfg.getFragmentizerThrottlingBlockLength();
- fragmentizerThrottlingDelay = cfg.getFragmentizerThrottlingDelay();
- secondaryFs = cfg.getSecondaryFileSystem();
- ipcEndpointCfg = cfg.getIpcEndpointConfiguration();
- ipcEndpointEnabled = cfg.isIpcEndpointEnabled();
- maxSpace = cfg.getMaxSpaceSize();
- maxTaskRangeLen = cfg.getMaximumTaskRangeLength();
- metaCacheName = cfg.getMetaCacheName();
- mgmtPort = cfg.getManagementPort();
- name = cfg.getName();
- pathModes = cfg.getPathModes();
- perNodeBatchSize = cfg.getPerNodeBatchSize();
- perNodeParallelBatchCnt = cfg.getPerNodeParallelBatchCount();
- prefetchBlocks = cfg.getPrefetchBlocks();
- seqReadsBeforePrefetch = cfg.getSequentialReadsBeforePrefetch();
- trashPurgeTimeout = cfg.getTrashPurgeTimeout();
- }
-
- /**
- * Gets IGFS instance name. If {@code null}, then instance with default
- * name will be used.
- *
- * @return IGFS instance name.
- */
- @Nullable public String getName() {
- return name;
- }
-
- /**
- * Sets IGFS instance name.
- *
- * @param name IGFS instance name.
- */
- public void setName(String name) {
- this.name = name;
- }
-
- /**
- * Cache name to store IGFS meta information. If {@code null}, then instance
- * with default meta-cache name will be used.
- *
- * @return Cache name to store IGFS meta information.
- */
- @Nullable public String getMetaCacheName() {
- return metaCacheName;
- }
-
- /**
- * Sets cache name to store IGFS meta information.
- *
- * @param metaCacheName Cache name to store IGFS meta information.
- */
- public void setMetaCacheName(String metaCacheName) {
- this.metaCacheName = metaCacheName;
- }
-
- /**
- * Cache name to store IGFS data.
- *
- * @return Cache name to store IGFS data.
- */
- @Nullable public String getDataCacheName() {
- return dataCacheName;
- }
-
- /**
- * Sets cache name to store IGFS data.
- *
- * @param dataCacheName Cache name to store IGFS data.
- */
- public void setDataCacheName(String dataCacheName) {
- this.dataCacheName = dataCacheName;
- }
-
- /**
- * Get file's data block size.
- *
- * @return File's data block size.
- */
- public int getBlockSize() {
- return blockSize;
- }
-
- /**
- * Sets file's data block size.
- *
- * @param blockSize File's data block size (bytes) or {@code 0} to reset default value.
- */
- public void setBlockSize(int blockSize) {
- A.ensure(blockSize >= 0, "blockSize >= 0");
-
- this.blockSize = blockSize == 0 ? DFLT_BLOCK_SIZE : blockSize;
- }
-
- /**
- * Get number of pre-fetched blocks if specific file's chunk is requested.
- *
- * @return The number of pre-fetched blocks.
- */
- public int getPrefetchBlocks() {
- return prefetchBlocks;
- }
-
- /**
- * Sets the number of pre-fetched blocks if specific file's chunk is requested.
- *
- * @param prefetchBlocks New number of pre-fetched blocks.
- */
- public void setPrefetchBlocks(int prefetchBlocks) {
- A.ensure(prefetchBlocks >= 0, "prefetchBlocks >= 0");
-
- this.prefetchBlocks = prefetchBlocks;
- }
-
- /**
- * Get amount of sequential block reads before prefetch is triggered. The
- * higher this value, the longer IGFS will wait before starting to prefetch
- * values ahead of time. Depending on the use case, this can either help
- * or hurt performance.
- * <p>
- * Default is {@code 0} which means that pre-fetching will start right away.
- * <h1 class="header">Integration With Hadoop</h1>
- * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
- * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} configuration property directly to Hadoop
- * MapReduce task.
- * <p>
- * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- *
- * @return Amount of sequential block reads.
- */
- public int getSequentialReadsBeforePrefetch() {
- return seqReadsBeforePrefetch;
- }
-
- /**
- * Sets amount of sequential block reads before prefetch is triggered. The
- * higher this value, the longer IGFS will wait before starting to prefetch
- * values ahead of time. Depending on the use case, this can either help
- * or hurt performance.
- * <p>
- * Default is {@code 0} which means that pre-fetching will start right away.
- * <h1 class="header">Integration With Hadoop</h1>
- * This parameter can be also overridden for individual Hadoop MapReduce tasks by passing
- * {@code fs.igfs.[name].open.sequential_reads_before_prefetch} configuration property directly to Hadoop
- * MapReduce task.
- * <p>
- * <b>NOTE:</b> Integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- *
- * @param seqReadsBeforePrefetch Amount of sequential block reads before prefetch is triggered.
- */
- public void setSequentialReadsBeforePrefetch(int seqReadsBeforePrefetch) {
- A.ensure(seqReadsBeforePrefetch >= 0, "seqReadsBeforePrefetch >= 0");
-
- this.seqReadsBeforePrefetch = seqReadsBeforePrefetch;
- }
-
- /**
- * Get read/write buffer size for {@code IGFS} stream operations in bytes.
- *
- * @return Read/write buffers size (bytes).
- */
- public int getStreamBufferSize() {
- return bufSize;
- }
-
- /**
- * Sets read/write buffers size for {@code IGFS} stream operations (bytes).
- *
- * @param bufSize Read/write buffers size for stream operations (bytes) or {@code 0} to reset default value.
- */
- public void setStreamBufferSize(int bufSize) {
- A.ensure(bufSize >= 0, "bufSize >= 0");
-
- this.bufSize = bufSize == 0 ? DFLT_BUF_SIZE : bufSize;
- }
-
- /**
- * Gets number of file blocks buffered on local node before sending batch to remote node.
- *
- * @return Per node buffer size.
- */
- public int getPerNodeBatchSize() {
- return perNodeBatchSize;
- }
-
- /**
- * Sets number of file blocks collected on local node before sending batch to remote node.
- *
- * @param perNodeBatchSize Per node buffer size.
- */
- public void setPerNodeBatchSize(int perNodeBatchSize) {
- this.perNodeBatchSize = perNodeBatchSize;
- }
-
- /**
- * Gets number of batches that can be concurrently sent to remote node.
- *
- * @return Number of batches for each node.
- */
- public int getPerNodeParallelBatchCount() {
- return perNodeParallelBatchCnt;
- }
-
- /**
- * Sets number of file block batches that can be concurrently sent to remote node.
- *
- * @param perNodeParallelBatchCnt Per node parallel load operations.
- */
- public void setPerNodeParallelBatchCount(int perNodeParallelBatchCnt) {
- this.perNodeParallelBatchCnt = perNodeParallelBatchCnt;
- }
-
- /**
- * Gets map of IPC endpoint configuration properties. There are 2 different
- * types of endpoint supported: {@code shared-memory}, and {@code TCP}.
- * <p>
- * The following configuration properties are supported for {@code shared-memory}
- * endpoint:
- * <ul>
- * <li>{@code type} - value is {@code shmem} to specify {@code shared-memory} approach.</li>
- * <li>{@code port} - endpoint port.</li>
- * <li>{@code size} - memory size allocated for single endpoint communication.</li>
- * <li>
- * {@code tokenDirectoryPath} - path, either absolute or relative to {@code IGNITE_HOME} to
- * store shared memory tokens.
- * </li>
- * </ul>
- * <p>
- * The following configuration properties are supported for {@code TCP} approach:
- * <ul>
- * <li>{@code type} - value is {@code tcp} to specify {@code TCP} approach.</li>
- * <li>{@code port} - endpoint bind port.</li>
- * <li>
- * {@code host} - endpoint bind host. If omitted '127.0.0.1' will be used.
- * </li>
- * </ul>
- * <p>
- * Note that {@code shared-memory} approach is not supported on Windows environments.
- * In case IGFS is failed to bind to particular port, further attempts will be performed every 3 seconds.
- *
- * @return Map of IPC endpoint configuration properties. In case the value is not set, defaults will be used. Default
- * type for Windows is "tcp", for all other platforms - "shmem". Default port is {@link #DFLT_IPC_PORT}.
- */
- @Nullable public Map<String,String> getIpcEndpointConfiguration() {
- return ipcEndpointCfg;
- }
-
- /**
- * Sets IPC endpoint configuration to publish IGFS over.
- *
- * @param ipcEndpointCfg Map of IPC endpoint config properties.
- */
- public void setIpcEndpointConfiguration(@Nullable Map<String,String> ipcEndpointCfg) {
- this.ipcEndpointCfg = ipcEndpointCfg;
- }
-
- /**
- * Get IPC endpoint enabled flag. In case it is set to {@code true} endpoint will be created and bound to specific
- * port. Otherwise endpoint will not be created. Default value is {@link #DFLT_IPC_ENDPOINT_ENABLED}.
- *
- * @return {@code True} in case endpoint is enabled.
- */
- public boolean isIpcEndpointEnabled() {
- return ipcEndpointEnabled;
- }
-
- /**
- * Set IPC endpoint enabled flag. See {@link #isIpcEndpointEnabled()}.
- *
- * @param ipcEndpointEnabled IPC endpoint enabled flag.
- */
- public void setIpcEndpointEnabled(boolean ipcEndpointEnabled) {
- this.ipcEndpointEnabled = ipcEndpointEnabled;
- }
-
- /**
- * Gets port number for management endpoint. All IGFS nodes should have this port open
- * for Visor Management Console to work with IGFS.
- * <p>
- * Default value is {@link #DFLT_MGMT_PORT}
- *
- * @return Port number or {@code -1} if management endpoint should be disabled.
- */
- public int getManagementPort() {
- return mgmtPort;
- }
-
- /**
- * Sets management endpoint port.
- *
- * @param mgmtPort port number or {@code -1} to disable management endpoint.
- */
- public void setManagementPort(int mgmtPort) {
- this.mgmtPort = mgmtPort;
- }
-
- /**
- * Gets mode to specify how {@code IGFS} interacts with Hadoop file system, like {@code HDFS}.
- * Secondary Hadoop file system is provided for pass-through, write-through, and read-through
- * purposes.
- * <p>
- * Default mode is {@link org.apache.ignite.igfs.IgfsMode#DUAL_ASYNC}. If secondary Hadoop file system is
- * not configured, this mode will work just like {@link org.apache.ignite.igfs.IgfsMode#PRIMARY} mode.
- *
- * @return Mode to specify how IGFS interacts with secondary HDFS file system.
- */
- public IgfsMode getDefaultMode() {
- return dfltMode;
- }
-
- /**
- * Sets {@code IGFS} mode to specify how it should interact with secondary
- * Hadoop file system, like {@code HDFS}. Secondary Hadoop file system is provided
- * for pass-through, write-through, and read-through purposes.
- *
- * @param dfltMode {@code IGFS} mode.
- */
- public void setDefaultMode(IgfsMode dfltMode) {
- this.dfltMode = dfltMode;
- }
-
- /**
- * Gets the secondary file system. Secondary file system is provided for pass-through, write-through,
- * and read-through purposes.
- *
- * @return Secondary file system.
- */
- public SecondaryFileSystem getSecondaryFileSystem() {
- return secondaryFs;
- }
-
- /**
- * Sets the secondary file system. Secondary file system is provided for pass-through, write-through,
- * and read-through purposes.
- *
- * @param fileSystem
- */
- public void setSecondaryFileSystem(SecondaryFileSystem fileSystem) {
- secondaryFs = fileSystem;
- }
-
- /**
- * Gets map of path prefixes to {@code IGFS} modes used for them.
- * <p>
- * If path doesn't correspond to any specified prefix or mappings are not provided, then
- * {@link #getDefaultMode()} is used.
- * <p>
- * Several folders under {@code '/apache/ignite'} folder have predefined mappings which cannot be overridden.
- * <li>{@code /apache/ignite/primary} and all it's sub-folders will always work in {@code PRIMARY} mode.</li>
- * <p>
- * And in case secondary file system URI is provided:
- * <li>{@code /apache/ignite/proxy} and all it's sub-folders will always work in {@code PROXY} mode.</li>
- * <li>{@code /apache/ignite/sync} and all it's sub-folders will always work in {@code DUAL_SYNC} mode.</li>
- * <li>{@code /apache/ignite/async} and all it's sub-folders will always work in {@code DUAL_ASYNC} mode.</li>
- *
- * @return Map of paths to {@code IGFS} modes.
- */
- @Nullable public Map<String, IgfsMode> getPathModes() {
- return pathModes;
- }
-
- /**
- * Sets map of path prefixes to {@code IGFS} modes used for them.
- * <p>
- * If path doesn't correspond to any specified prefix or mappings are not provided, then
- * {@link #getDefaultMode()} is used.
- *
- * @param pathModes Map of paths to {@code IGFS} modes.
- */
- public void setPathModes(Map<String, IgfsMode> pathModes) {
- this.pathModes = pathModes;
- }
-
- /**
- * Gets the length of file chunk to send before delaying the fragmentizer.
- *
- * @return File chunk length in bytes.
- */
- public long getFragmentizerThrottlingBlockLength() {
- return fragmentizerThrottlingBlockLen;
- }
-
- /**
- * Sets length of file chunk to transmit before throttling is delayed.
- *
- * @param fragmentizerThrottlingBlockLen Block length in bytes.
- */
- public void setFragmentizerThrottlingBlockLength(long fragmentizerThrottlingBlockLen) {
- this.fragmentizerThrottlingBlockLen = fragmentizerThrottlingBlockLen;
- }
-
- /**
- * Gets throttle delay for fragmentizer.
- *
- * @return Throttle delay in milliseconds.
- */
- public long getFragmentizerThrottlingDelay() {
- return fragmentizerThrottlingDelay;
- }
-
- /**
- * Sets delay in milliseconds for which fragmentizer is paused.
- *
- * @param fragmentizerThrottlingDelay Delay in milliseconds.
- */
- public void setFragmentizerThrottlingDelay(long fragmentizerThrottlingDelay) {
- this.fragmentizerThrottlingDelay = fragmentizerThrottlingDelay;
- }
-
- /**
- * Gets number of files that can be processed by fragmentizer concurrently.
- *
- * @return Number of files to process concurrently.
- */
- public int getFragmentizerConcurrentFiles() {
- return fragmentizerConcurrentFiles;
- }
-
- /**
- * Sets number of files to process concurrently by fragmentizer.
- *
- * @param fragmentizerConcurrentFiles Number of files to process concurrently.
- */
- public void setFragmentizerConcurrentFiles(int fragmentizerConcurrentFiles) {
- this.fragmentizerConcurrentFiles = fragmentizerConcurrentFiles;
- }
-
- /**
- * Gets amount of local memory (in % of local IGFS max space size) available for local writes
- * during file creation.
- * <p>
- * If current IGFS space size is less than {@code fragmentizerLocalWritesRatio * maxSpaceSize},
- * then file blocks will be written to the local node first and then asynchronously distributed
- * among cluster nodes (fragmentized).
- * <p>
- * Default value is {@link #DFLT_FRAGMENTIZER_LOCAL_WRITES_RATIO}.
- *
- * @return Ratio for local writes space.
- */
- public float getFragmentizerLocalWritesRatio() {
- return fragmentizerLocWritesRatio;
- }
-
- /**
- * Sets ratio for space available for local file writes.
- *
- * @param fragmentizerLocWritesRatio Ratio for local file writes.
- * @see #getFragmentizerLocalWritesRatio()
- */
- public void setFragmentizerLocalWritesRatio(float fragmentizerLocWritesRatio) {
- this.fragmentizerLocWritesRatio = fragmentizerLocWritesRatio;
- }
-
- /**
- * Gets flag indicating whether IGFS fragmentizer is enabled. If fragmentizer is disabled, files will be
- * written in distributed fashion.
- *
- * @return Flag indicating whether fragmentizer is enabled.
- */
- public boolean isFragmentizerEnabled() {
- return fragmentizerEnabled;
- }
-
- /**
- * Sets property indicating whether fragmentizer is enabled.
- *
- * @param fragmentizerEnabled {@code True} if fragmentizer is enabled.
- */
- public void setFragmentizerEnabled(boolean fragmentizerEnabled) {
- this.fragmentizerEnabled = fragmentizerEnabled;
- }
-
- /**
- * Get maximum space available for data cache to store file system entries.
- *
- * @return Maximum space available for data cache.
- */
- public long getMaxSpaceSize() {
- return maxSpace;
- }
-
- /**
- * Set maximum space in bytes available in data cache.
- *
- * @param maxSpace Maximum space available in data cache.
- */
- public void setMaxSpaceSize(long maxSpace) {
- this.maxSpace = maxSpace;
- }
-
- /**
- * Gets maximum timeout awaiting for trash purging in case data cache oversize is detected.
- *
- * @return Maximum timeout awaiting for trash purging in case data cache oversize is detected.
- */
- public long getTrashPurgeTimeout() {
- return trashPurgeTimeout;
- }
-
- /**
- * Sets maximum timeout awaiting for trash purging in case data cache oversize is detected.
- *
- * @param trashPurgeTimeout Maximum timeout awaiting for trash purging in case data cache oversize is detected.
- */
- public void setTrashPurgeTimeout(long trashPurgeTimeout) {
- this.trashPurgeTimeout = trashPurgeTimeout;
- }
-
- /**
- * Get DUAL mode put operation executor service. This executor service will process cache PUT requests for
- * data which came from the secondary file system and about to be written to IGFS data cache.
- * In case no executor service is provided, default one will be created with maximum amount of threads equals
- * to amount of processor cores.
- *
- * @return Get DUAL mode put operation executor service
- */
- @Nullable public ExecutorService getDualModePutExecutorService() {
- return dualModePutExec;
- }
-
- /**
- * Set DUAL mode put operations executor service.
- *
- * @param dualModePutExec Dual mode put operations executor service.
- */
- public void setDualModePutExecutorService(ExecutorService dualModePutExec) {
- this.dualModePutExec = dualModePutExec;
- }
-
- /**
- * Get DUAL mode put operation executor service shutdown flag.
- *
- * @return DUAL mode put operation executor service shutdown flag.
- */
- public boolean getDualModePutExecutorServiceShutdown() {
- return dualModePutExecShutdown;
- }
-
- /**
- * Set DUAL mode put operations executor service shutdown flag.
- *
- * @param dualModePutExecShutdown Dual mode put operations executor service shutdown flag.
- */
- public void setDualModePutExecutorServiceShutdown(boolean dualModePutExecShutdown) {
- this.dualModePutExecShutdown = dualModePutExecShutdown;
- }
-
- /**
- * Get maximum amount of pending data read from the secondary file system and waiting to be written to data
- * cache. {@code 0} or negative value stands for unlimited size.
- * <p>
- * By default this value is set to {@code 0}. It is recommended to set positive value in case your
- * application performs frequent reads of large amount of data from the secondary file system in order to
- * avoid issues with increasing GC pauses or out-of-memory error.
- *
- * @return Maximum amount of pending data read from the secondary file system
- */
- public long getDualModeMaxPendingPutsSize() {
- return dualModeMaxPendingPutsSize;
- }
-
- /**
- * Set maximum amount of data in pending put operations.
- *
- * @param dualModeMaxPendingPutsSize Maximum amount of data in pending put operations.
- */
- public void setDualModeMaxPendingPutsSize(long dualModeMaxPendingPutsSize) {
- this.dualModeMaxPendingPutsSize = dualModeMaxPendingPutsSize;
- }
-
- /**
- * Get maximum default range size of a file being split during IGFS task execution. When IGFS task is about to
- * be executed, it requests file block locations first. Each location is defined as {@link org.apache.ignite.igfs.mapreduce.IgfsFileRange} which
- * has length. In case this parameter is set to positive value, then IGFS will split single file range into smaller
- * ranges with length not greater that this parameter. The only exception to this case is when maximum task range
- * length is smaller than file block size. In this case maximum task range size will be overridden and set to file
- * block size.
- * <p>
- * Note that this parameter is applied when task is split into jobs before {@link org.apache.ignite.igfs.mapreduce.IgfsRecordResolver} is
- * applied. Therefore, final file ranges being assigned to particular jobs could be greater than value of this
- * parameter depending on file data layout and selected resolver type.
- * <p>
- * Setting this parameter might be useful when file is highly colocated and have very long consequent data chunks
- * so that task execution suffers from insufficient parallelism. E.g., in case you have one IGFS node in topology
- * and want to process 1Gb file, then only single range of length 1Gb will be returned. This will result in
- * a single job which will be processed in one thread. But in case you provide this configuration parameter and set
- * maximum range length to 16Mb, then 64 ranges will be returned resulting in 64 jobs which could be executed in
- * parallel.
- * <p>
- * Note that some {@code IgniteFs.execute()} methods can override value of this parameter.
- * <p>
- * In case value of this parameter is set to {@code 0} or negative value, it is simply ignored. Default value is
- * {@code 0}.
- *
- * @return Maximum range size of a file being split during IGFS task execution.
- */
- public long getMaximumTaskRangeLength() {
- return maxTaskRangeLen;
- }
-
- /**
- * Set maximum default range size of a file being split during IGFS task execution.
- * See {@link #getMaximumTaskRangeLength()} for more details.
- *
- * @param maxTaskRangeLen Set maximum default range size of a file being split during IGFS task execution.
- */
- public void setMaximumTaskRangeLength(long maxTaskRangeLen) {
- this.maxTaskRangeLen = maxTaskRangeLen;
- }
-
- /** {@inheritDoc} */
- @Override public String toString() {
- return S.toString(FileSystemConfiguration.class, this);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite_new/filesystem/SecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite_new/filesystem/SecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite_new/filesystem/SecondaryFileSystem.java
deleted file mode 100644
index ae86c2b..0000000
--- a/modules/core/src/main/java/org/apache/ignite_new/filesystem/SecondaryFileSystem.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite_new.filesystem;
-
-import org.apache.ignite.*;
-import org.apache.ignite.igfs.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Secondary file system interface.
- */
-public interface SecondaryFileSystem {
- /** File property: user name. */
- public static final String PROP_USER_NAME = "usrName";
-
- /** File property: group name. */
- public static final String PROP_GROUP_NAME = "grpName";
-
- /** File property: permission. */
- public static final String PROP_PERMISSION = "permission";
-
- /**
- * Checks if the specified path exists in the file system.
- *
- * @param path Path to check for existence in the file system.
- * @return {@code True} if such file exists, otherwise - {@code false}.
- * @throws IgniteException In case of error.
- */
- public boolean exists(IgfsPath path);
-
- /**
- * Updates file information for the specified path. Existent properties, not listed in the passed collection,
- * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
- * will be removed from the stored properties or ignored if they don't exist in the file info.
- * <p>
- * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
- * to the secondary file system:
- * <ul>
- * <li>{@code usrName} - file owner name;</li>
- * <li>{@code grpName} - file owner group;</li>
- * <li>{@code permission} - Unix-style string representing file permissions.</li>
- * </ul>
- *
- * @param path File path to set properties for.
- * @param props Properties to update.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
-
- /**
- * Renames/moves a file.
- * <p>
- * You are free to rename/move data files as you wish, but directories can be only renamed.
- * You cannot move the directory between different parent directories.
- * <p>
- * Examples:
- * <ul>
- * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
- * <li>"/work" => "/work-2012.bkp"</li>
- * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
- * </ul>
- *
- * @param src Source file path to rename.
- * @param dest Destination file path. If destination path is a directory, then source file will be placed
- * into destination directory with original name.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If source file doesn't exist.
- */
- public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
-
- /**
- * Deletes file.
- *
- * @param path File path to delete.
- * @param recursive Delete non-empty directories recursively.
- * @return {@code True} in case of success, {@code false} otherwise.
- * @throws IgniteException In case of error.
- */
- public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
-
- /**
- * Creates directories under specified path.
- *
- * @param path Path of directories chain to create.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path) throws IgniteException;
-
- /**
- * Creates directories under specified path with the specified properties.
- *
- * @param path Path of directories chain to create.
- * @param props Metadata properties to set on created directories.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Lists file paths under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
-
- /**
- * Lists files under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsReader open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Gets file information for the specified path.
- *
- * @param path Path to get information for.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile info(IgfsPath path) throws IgniteException;
-
- /**
- * Gets used space in bytes.
- *
- * @return Used space in bytes.
- * @throws IgniteException In case of error.
- */
- public long usedSpaceSize() throws IgniteException;
-
- /**
- * Gets the implementation specific properties of file system.
- *
- * @return Map of properties.
- */
- public Map<String,String> properties();
-}
[3/6] incubator-ignite git commit: # IGNITE-386: WIP.
Posted by vo...@apache.org.
# IGNITE-386: WIP.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/dd06414a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/dd06414a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/dd06414a
Branch: refs/heads/ignite-386
Commit: dd06414a22a5379ac93dc505a04fdca2212da988
Parents: 28fad18
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Mar 3 17:43:01 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Mar 3 17:43:01 2015 +0300
----------------------------------------------------------------------
config/hadoop/default-config.xml | 2 +-
.../ignite/examples/igfs/IgfsExample.java | 18 +-
.../examples/igfs/IgfsMapReduceExample.java | 6 +-
.../src/main/java/org/apache/ignite/Ignite.java | 6 +-
.../org/apache/ignite/IgniteFileSystem.java | 347 ++++++++
.../main/java/org/apache/ignite/IgniteFs.java | 346 --------
.../ignite/configuration/IgfsConfiguration.java | 7 +-
.../main/java/org/apache/ignite/igfs/Igfs.java | 216 -----
.../apache/ignite/igfs/IgfsBlockLocation.java | 2 +-
.../java/org/apache/ignite/igfs/IgfsFile.java | 2 +-
.../org/apache/ignite/igfs/IgfsInputStream.java | 4 +-
.../java/org/apache/ignite/igfs/IgfsReader.java | 38 -
.../mapreduce/IgfsInputStreamJobAdapter.java | 4 +-
.../apache/ignite/igfs/mapreduce/IgfsJob.java | 4 +-
.../igfs/mapreduce/IgfsRecordResolver.java | 2 +-
.../apache/ignite/igfs/mapreduce/IgfsTask.java | 2 +-
.../IgfsByteDelimiterRecordResolver.java | 3 +-
.../records/IgfsFixedLengthRecordResolver.java | 2 +-
.../igfs/secondary/IgfsSecondaryFileSystem.java | 217 +++++
.../IgfsSecondaryFileSystemReader.java | 38 +
.../apache/ignite/igfs/secondary/package.html | 24 +
.../org/apache/ignite/internal/IgniteEx.java | 2 +-
.../apache/ignite/internal/IgniteKernal.java | 8 +-
.../internal/processors/igfs/IgfsAsyncImpl.java | 2 +-
.../processors/igfs/IgfsDataManager.java | 3 +-
.../ignite/internal/processors/igfs/IgfsEx.java | 4 +-
.../internal/processors/igfs/IgfsImpl.java | 9 +-
.../processors/igfs/IgfsInputStreamImpl.java | 5 +-
.../internal/processors/igfs/IgfsJobImpl.java | 2 +-
.../processors/igfs/IgfsMetaManager.java | 25 +-
.../processors/igfs/IgfsNoopProcessor.java | 4 +-
.../processors/igfs/IgfsOutputStreamImpl.java | 2 +-
.../internal/processors/igfs/IgfsProcessor.java | 8 +-
.../processors/igfs/IgfsProcessorAdapter.java | 4 +-
.../IgfsSecondaryInputStreamDescriptor.java | 8 +-
.../ignite/internal/visor/igfs/VisorIgfs.java | 4 +-
.../visor/igfs/VisorIgfsProfilerClearTask.java | 2 +-
.../visor/node/VisorIgfsConfiguration.java | 3 +-
.../visor/node/VisorNodeDataCollectorJob.java | 2 +-
.../internal/visor/util/VisorTaskUtils.java | 2 +-
.../org/apache/ignite_new/IgniteFileSystem.java | 505 ------------
.../configuration/FileSystemConfiguration.java | 809 -------------------
.../filesystem/SecondaryFileSystem.java | 210 -----
.../ignite/igfs/IgfsEventsAbstractSelfTest.java | 2 +-
.../ignite/igfs/IgfsFragmentizerSelfTest.java | 8 +-
.../igfs/IgfsFragmentizerTopologySelfTest.java | 2 +-
...heIgfsPerBlockLruEvictionPolicySelfTest.java | 2 +-
.../processors/igfs/IgfsAbstractSelfTest.java | 19 +-
.../igfs/IgfsDualAbstractSelfTest.java | 2 +-
.../processors/igfs/IgfsMetricsSelfTest.java | 10 +-
.../processors/igfs/IgfsProcessorSelfTest.java | 4 +-
.../processors/igfs/IgfsStreamsSelfTest.java | 10 +-
.../processors/igfs/IgfsTaskSelfTest.java | 6 +-
.../IgfsAbstractRecordResolverSelfTest.java | 2 +-
.../ignite/testframework/junits/IgniteMock.java | 4 +-
.../fs/IgniteHadoopIgfsSecondaryFileSystem.java | 415 ++++++++++
.../fs/IgniteHadoopSecondaryFileSystem.java | 413 ----------
.../hadoop/fs/v1/IgniteHadoopFileSystem.java | 6 +-
.../hadoop/fs/v2/IgniteHadoopFileSystem.java | 6 +-
.../hadoop/igfs/HadoopIgfsEndpoint.java | 2 +-
.../hadoop/igfs/HadoopIgfsProperties.java | 2 +-
.../hadoop/igfs/HadoopIgfsReader.java | 104 ---
.../HadoopIgfsSecondaryFileSystemReader.java | 104 +++
.../planner/HadoopDefaultMapReducePlanner.java | 2 +-
.../hadoop/HadoopClientProtocolSelfTest.java | 6 +-
.../HadoopIgfs20FileSystemAbstractSelfTest.java | 6 +-
.../igfs/HadoopIgfsDualAbstractSelfTest.java | 5 +-
.../apache/ignite/igfs/IgfsEventsTestSuite.java | 8 +-
.../IgniteHadoopFileSystemAbstractSelfTest.java | 4 +-
...IgniteHadoopFileSystemHandshakeSelfTest.java | 2 +-
...teHadoopFileSystemSecondaryModeSelfTest.java | 2 +-
.../HadoopDefaultMapReducePlannerSelfTest.java | 6 +-
.../hadoop/HadoopTaskExecutionSelfTest.java | 2 +-
.../HadoopExternalTaskExecutionSelfTest.java | 2 +-
.../org/apache/ignite/IgniteSpringBean.java | 4 +-
pom.xml | 4 +-
76 files changed, 1301 insertions(+), 2788 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/config/hadoop/default-config.xml
----------------------------------------------------------------------
diff --git a/config/hadoop/default-config.xml b/config/hadoop/default-config.xml
index f500529..bb35b03 100644
--- a/config/hadoop/default-config.xml
+++ b/config/hadoop/default-config.xml
@@ -135,7 +135,7 @@
<property name="defaultMode" value="PROXY"/>
<property name="secondaryFileSystem">
- <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopSecondaryFileSystem">
+ <bean class="org.apache.ignite.hadoop.fs.IgniteHadoopIgfsSecondaryFileSystem">
<constructor-arg name="uri" value="hdfs://1.2.3.4:9000"/>
<constructor-arg name="cfgPath" value="/opt/hadoop-server/etc/hadoop/core-site.xml"/>
</bean>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
index 56f3fa4..8bfd38d 100644
--- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsExample.java
@@ -25,7 +25,7 @@ import java.io.*;
import java.util.*;
/**
- * Example that shows usage of {@link org.apache.ignite.IgniteFs} API. It starts a node with {@code IgniteFs}
+ * Example that shows usage of {@link org.apache.ignite.IgniteFileSystem} API. It starts a node with {@code IgniteFs}
* configured and performs several file system operations (create, write, append, read and delete
* files, create, list and delete directories).
* <p>
@@ -50,7 +50,7 @@ public final class IgfsExample {
try {
// Get an instance of Ignite File System.
- IgniteFs fs = ignite.fileSystem("igfs");
+ IgniteFileSystem fs = ignite.fileSystem("igfs");
// Working directory path.
IgfsPath workDir = new IgfsPath("/examples/fs");
@@ -107,7 +107,7 @@ public final class IgfsExample {
* @param path File or directory path.
* @throws IgniteException In case of error.
*/
- private static void delete(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void delete(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
@@ -139,7 +139,7 @@ public final class IgfsExample {
* @param path Directory path.
* @throws IgniteException In case of error.
*/
- private static void mkdirs(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void mkdirs(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
@@ -166,7 +166,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be created.
* @throws IOException If data can't be written.
*/
- private static void create(IgniteFs fs, IgfsPath path, @Nullable byte[] data)
+ private static void create(IgniteFileSystem fs, IgfsPath path, @Nullable byte[] data)
throws IgniteException, IOException {
assert fs != null;
assert path != null;
@@ -195,7 +195,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be created.
* @throws IOException If data can't be written.
*/
- private static void append(IgniteFs fs, IgfsPath path, byte[] data) throws IgniteException, IOException {
+ private static void append(IgniteFileSystem fs, IgfsPath path, byte[] data) throws IgniteException, IOException {
assert fs != null;
assert path != null;
assert data != null;
@@ -220,7 +220,7 @@ public final class IgfsExample {
* @throws IgniteException If file can't be opened.
* @throws IOException If data can't be read.
*/
- private static void read(IgniteFs fs, IgfsPath path) throws IgniteException, IOException {
+ private static void read(IgniteFileSystem fs, IgfsPath path) throws IgniteException, IOException {
assert fs != null;
assert path != null;
assert fs.info(path).isFile();
@@ -242,7 +242,7 @@ public final class IgfsExample {
* @param path Directory path.
* @throws IgniteException In case of error.
*/
- private static void list(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void list(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
assert fs != null;
assert path != null;
assert fs.info(path).isDirectory();
@@ -271,7 +271,7 @@ public final class IgfsExample {
* @param path File or directory path.
* @throws IgniteException In case of error.
*/
- private static void printInfo(IgniteFs fs, IgfsPath path) throws IgniteException {
+ private static void printInfo(IgniteFileSystem fs, IgfsPath path) throws IgniteException {
System.out.println();
System.out.println("Information for " + path + ": " + fs.info(path));
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
index 22a66a3..ed0abe4 100644
--- a/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
+++ b/examples/src/main/java/org/apache/ignite/examples/igfs/IgfsMapReduceExample.java
@@ -61,7 +61,7 @@ public class IgfsMapReduceExample {
String regexStr = args[1];
// Get an instance of Ignite File System.
- IgniteFs fs = ignite.fileSystem("igfs");
+ IgniteFileSystem fs = ignite.fileSystem("igfs");
// Working directory path.
IgfsPath workDir = new IgfsPath("/examples/fs");
@@ -94,7 +94,7 @@ public class IgfsMapReduceExample {
* @param file File to write.
* @throws Exception In case of exception.
*/
- private static void writeFile(IgniteFs fs, IgfsPath fsPath, File file) throws Exception {
+ private static void writeFile(IgniteFileSystem fs, IgfsPath fsPath, File file) throws Exception {
System.out.println();
System.out.println("Copying file to IGFS: " + file);
@@ -173,7 +173,7 @@ public class IgfsMapReduceExample {
}
/** {@inheritDoc} */
- @Override public Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException {
+ @Override public Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException {
Collection<Line> res = null;
long start = in.startOffset();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/Ignite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/Ignite.java b/modules/core/src/main/java/org/apache/ignite/Ignite.java
index 8851d8f..7477690 100644
--- a/modules/core/src/main/java/org/apache/ignite/Ignite.java
+++ b/modules/core/src/main/java/org/apache/ignite/Ignite.java
@@ -43,7 +43,7 @@ import java.util.concurrent.*;
* <ul>
* <li>{@link org.apache.ignite.cache.GridCache} - functionality for in-memory distributed cache.</li>
* <li>{@link IgniteDataLoader} - functionality for loading data large amounts of data into cache.</li>
- * <li>{@link IgniteFs} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li>
+ * <li>{@link IgniteFileSystem} - functionality for distributed Hadoop-compliant in-memory file system and map-reduce.</li>
* <li>{@link IgniteStreamer} - functionality for streaming events workflow with queries and indexes into rolling windows.</li>
* <li>{@link IgniteScheduler} - functionality for scheduling jobs using UNIX Cron syntax.</li>
* <li>{@link IgniteCompute} - functionality for executing tasks and closures on all grid nodes (inherited form {@link ClusterGroup}).</li>
@@ -223,14 +223,14 @@ public interface Ignite extends AutoCloseable {
* @param name IGFS name.
* @return IGFS instance.
*/
- public IgniteFs fileSystem(String name);
+ public IgniteFileSystem fileSystem(String name);
/**
* Gets all instances of the grid file systems.
*
* @return Collection of grid file systems instances.
*/
- public Collection<IgniteFs> fileSystems();
+ public Collection<IgniteFileSystem> fileSystems();
/**
* Gets an instance of streamer by name, if one does not exist then
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
new file mode 100644
index 0000000..893bca8
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -0,0 +1,347 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite;
+
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.apache.ignite.lang.*;
+import org.jetbrains.annotations.*;
+
+import java.util.*;
+
+/**
+ * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
+ * <ul>
+ * <li>list directories or get information for a single path</li>
+ * <li>create/move/delete files or directories</li>
+ * <li>write/read data streams into/from files</li>
+ * </ul>
+ * The data of each file is split on separate data blocks and stored in the cache.
+ * You can access file's data with a standard Java streaming API. Moreover, for each part
+ * of the file you can calculate an affinity and process file's content on corresponding
+ * nodes to escape unnecessary networking.
+ * <p/>
+ * This API is fully thread-safe and you can use it from several threads.
+ * <h1 class="header">IGFS Configuration</h1>
+ * The simplest way to run a Ignite node with configured file system is to pass
+ * special configuration file included in Ignite distribution to {@code ignite.sh} or
+ * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml}
+ * <p>
+ * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for
+ * caching data, while client node is responsible for basic file system operations and accessing
+ * data nodes remotely. When used as Hadoop file system, clients nodes usually started together
+ * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
+ * started together with Hadoop {@code task-tracker} processes.
+ * <h1 class="header">Integration With Hadoop</h1>
+ * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by
+ * plugging in as {@code Hadoop FileSystem}. Refer to
+ * {@code org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem} or
+ * {@code org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem} for more information.
+ * <p>
+ * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
+ */
+public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSupport {
+ /** IGFS scheme name. */
+ public static final String IGFS_SCHEME = "igfs";
+
+ /** File property: prefer writes to local node. */
+ public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
+
+ /**
+ * Gets IGFS name.
+ *
+ * @return IGFS name, or {@code null} for default file system.
+ */
+ public String name();
+
+ /**
+ * Gets IGFS configuration.
+ *
+ * @return IGFS configuration.
+ */
+ public IgfsConfiguration configuration();
+
+ /**
+ * Gets summary (total number of files, total number of directories and total length)
+ * for a given path.
+ *
+ * @param path Path to get information for.
+ * @return Summary object.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found.
+ * @throws IgniteException If failed.
+ */
+ public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsInputStream open(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ @Override public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be
+ * stored on node where {@code affKey} resides.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
+ @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props)
+ throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param create Create file if it doesn't exist yet.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param create Create file if it doesn't exist yet.
+ * @param props File properties to set only in case it file was just created.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Sets last access time and last modification time for a given path. If argument is {@code null},
+ * corresponding time will not be changed.
+ *
+ * @param path Path to update.
+ * @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
+ * @param modificationTime Optional last modification time to set. Value {@code -1} does not update
+ * modification time.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found.
+ * @throws IgniteException If error occurred.
+ */
+ public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
+
+ /**
+ * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks
+ * are stored.
+ *
+ * @param path File path to get affinity for.
+ * @param start Position in the file to start affinity resolution from.
+ * @param len Size of data in the file to resolve affinity for.
+ * @return Affinity block locations.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
+
+ /**
+ * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and
+ * particular block location length is greater than this value, block locations will be split into smaller
+ * chunks.
+ *
+ * @param path File path to get affinity for.
+ * @param start Position in the file to start affinity resolution from.
+ * @param len Size of data in the file to resolve affinity for.
+ * @param maxLen Maximum length of a single returned block location length.
+ * @return Affinity block locations.
+ * @throws IgniteException In case of error.
+ * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
+ throws IgniteException;
+
+ /**
+ * Gets metrics snapshot for this file system.
+ *
+ * @return Metrics.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsMetrics metrics() throws IgniteException;
+
+ /**
+ * Resets metrics for this file system.
+ *
+ * @throws IgniteException In case of error.
+ */
+ public void resetMetrics() throws IgniteException;
+
+ /**
+ * Determines size of the file denoted by provided path. In case if path is a directory, then
+ * total size of all containing entries will be calculated recursively.
+ *
+ * @param path File system path.
+ * @return Total size.
+ * @throws IgniteException In case of error.
+ */
+ public long size(IgfsPath path) throws IgniteException;
+
+ /**
+ * Formats the file system removing all existing entries from it.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @throws IgniteException In case format has failed.
+ */
+ @IgniteAsyncSupported
+ public void format() throws IgniteException;
+
+ /**
+ * Executes IGFS task.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param task Task to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
+
+ /**
+ * Executes IGFS task with overridden maximum range length (see
+ * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param task Task to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
+ * be ignored. Otherwise an exception will be thrown.
+ * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive
+ * IGFS blocks will be included.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
+ Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg)
+ throws IgniteException;
+
+ /**
+ * Executes IGFS task.
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param taskCls Task class to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
+
+ /**
+ * Executes IGFS task with overridden maximum range length (see
+ * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
+ * <p>
+ * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
+ *
+ * @param taskCls Task class to execute.
+ * @param rslvr Optional resolver to control split boundaries.
+ * @param paths Collection of paths to be processed within this task.
+ * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
+ * be ignored. Otherwise an exception will be thrown.
+ * @param maxRangeLen Maximum range length.
+ * @param arg Optional task argument.
+ * @return Task result.
+ * @throws IgniteException If execution failed.
+ */
+ @IgniteAsyncSupported
+ public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
+ @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
+ long maxRangeLen, @Nullable T arg) throws IgniteException;
+
+ /** {@inheritDoc} */
+ @Override public IgniteFileSystem withAsync();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java b/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
deleted file mode 100644
index 83fba8e..0000000
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFs.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite;
-
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.lang.*;
-import org.jetbrains.annotations.*;
-
-import java.util.*;
-
-/**
- * <b>IG</b>nite <b>F</b>ile <b>S</b>ystem API. It provides a typical file system "view" on a particular cache:
- * <ul>
- * <li>list directories or get information for a single path</li>
- * <li>create/move/delete files or directories</li>
- * <li>write/read data streams into/from files</li>
- * </ul>
- * The data of each file is split on separate data blocks and stored in the cache.
- * You can access file's data with a standard Java streaming API. Moreover, for each part
- * of the file you can calculate an affinity and process file's content on corresponding
- * nodes to escape unnecessary networking.
- * <p/>
- * This API is fully thread-safe and you can use it from several threads.
- * <h1 class="header">IGFS Configuration</h1>
- * The simplest way to run a Ignite node with configured file system is to pass
- * special configuration file included in Ignite distribution to {@code ignite.sh} or
- * {@code ignite.bat} scripts, like this: {@code ignite.sh config/hadoop/default-config.xml}
- * <p>
- * {@code IGFS} can be started as a data node or as a client node. Data node is responsible for
- * caching data, while client node is responsible for basic file system operations and accessing
- * data nodes remotely. When used as Hadoop file system, clients nodes usually started together
- * with {@code job-submitter} or {@code job-scheduler} processes, while data nodes are usually
- * started together with Hadoop {@code task-tracker} processes.
- * <h1 class="header">Integration With Hadoop</h1>
- * In addition to direct file system API, {@code IGFS} can be integrated with {@code Hadoop} by
- * plugging in as {@code Hadoop FileSystem}. Refer to
- * {@code org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem} or
- * {@code org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem} for more information.
- * <p>
- * <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
- */
-public interface IgniteFs extends Igfs, IgniteAsyncSupport {
- /** IGFS scheme name. */
- public static final String IGFS_SCHEME = "igfs";
-
- /** File property: prefer writes to local node. */
- public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
-
- /**
- * Gets IGFS name.
- *
- * @return IGFS name, or {@code null} for default file system.
- */
- public String name();
-
- /**
- * Gets IGFS configuration.
- *
- * @return IGFS configuration.
- */
- public IgfsConfiguration configuration();
-
- /**
- * Gets summary (total number of files, total number of directories and total length)
- * for a given path.
- *
- * @param path Path to get information for.
- * @return Summary object.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path is not found.
- * @throws IgniteException If failed.
- */
- public IgfsPathSummary summary(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- @Override public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @param seqReadsBeforePrefetch Amount of sequential reads before prefetch is started.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsInputStream open(IgfsPath path, int bufSize, int seqReadsBeforePrefetch) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
- long blockSize, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param affKey Affinity key used to store file blocks. If not {@code null}, the whole file will be
- * stored on node where {@code affKey} resides.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite,
- @Nullable IgniteUuid affKey, int replication, long blockSize, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param create Create file if it doesn't exist yet.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public IgfsOutputStream append(IgfsPath path, boolean create) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Sets last access time and last modification time for a given path. If argument is {@code null},
- * corresponding time will not be changed.
- *
- * @param path Path to update.
- * @param accessTime Optional last access time to set. Value {@code -1} does not update access time.
- * @param modificationTime Optional last modification time to set. Value {@code -1} does not update
- * modification time.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If target was not found.
- * @throws IgniteException If error occurred.
- */
- public void setTimes(IgfsPath path, long accessTime, long modificationTime) throws IgniteException;
-
- /**
- * Gets affinity block locations for data blocks of the file, i.e. the nodes, on which the blocks
- * are stored.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len) throws IgniteException;
-
- /**
- * Get affinity block locations for data blocks of the file. In case {@code maxLen} parameter is set and
- * particular block location length is greater than this value, block locations will be split into smaller
- * chunks.
- *
- * @param path File path to get affinity for.
- * @param start Position in the file to start affinity resolution from.
- * @param len Size of data in the file to resolve affinity for.
- * @param maxLen Maximum length of a single returned block location length.
- * @return Affinity block locations.
- * @throws IgniteException In case of error.
- * @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsBlockLocation> affinity(IgfsPath path, long start, long len, long maxLen)
- throws IgniteException;
-
- /**
- * Gets metrics snapshot for this file system.
- *
- * @return Metrics.
- * @throws IgniteException In case of error.
- */
- public IgfsMetrics metrics() throws IgniteException;
-
- /**
- * Resets metrics for this file system.
- *
- * @throws IgniteException In case of error.
- */
- public void resetMetrics() throws IgniteException;
-
- /**
- * Determines size of the file denoted by provided path. In case if path is a directory, then
- * total size of all containing entries will be calculated recursively.
- *
- * @param path File system path.
- * @return Total size.
- * @throws IgniteException In case of error.
- */
- public long size(IgfsPath path) throws IgniteException;
-
- /**
- * Formats the file system removing all existing entries from it.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @throws IgniteException In case format has failed.
- */
- @IgniteAsyncSupported
- public void format() throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param task Task to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Optional maximum range length. If {@code 0}, then by default all consecutive
- * IGFS blocks will be included.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(IgfsTask<T, R> task, @Nullable IgfsRecordResolver rslvr,
- Collection<IgfsPath> paths, boolean skipNonExistentFiles, long maxRangeLen, @Nullable T arg)
- throws IgniteException;
-
- /**
- * Executes IGFS task.
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, @Nullable T arg) throws IgniteException;
-
- /**
- * Executes IGFS task with overridden maximum range length (see
- * {@link org.apache.ignite.configuration.IgfsConfiguration#getMaximumTaskRangeLength()} for more information).
- * <p>
- * Supports asynchronous execution (see {@link IgniteAsyncSupport}).
- *
- * @param taskCls Task class to execute.
- * @param rslvr Optional resolver to control split boundaries.
- * @param paths Collection of paths to be processed within this task.
- * @param skipNonExistentFiles Whether to skip non existent files. If set to {@code true} non-existent files will
- * be ignored. Otherwise an exception will be thrown.
- * @param maxRangeLen Maximum range length.
- * @param arg Optional task argument.
- * @return Task result.
- * @throws IgniteException If execution failed.
- */
- @IgniteAsyncSupported
- public <T, R> R execute(Class<? extends IgfsTask<T, R>> taskCls,
- @Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
- long maxRangeLen, @Nullable T arg) throws IgniteException;
-
- /** {@inheritDoc} */
- @Override public IgniteFs withAsync();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java b/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
index 2ad79fc..a5dbedf 100644
--- a/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
+++ b/modules/core/src/main/java/org/apache/ignite/configuration/IgfsConfiguration.java
@@ -18,6 +18,7 @@
package org.apache.ignite.configuration;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.jetbrains.annotations.*;
@@ -121,7 +122,7 @@ public class IgfsConfiguration {
private int mgmtPort = DFLT_MGMT_PORT;
/** Secondary file system */
- private Igfs secondaryFs;
+ private IgfsSecondaryFileSystem secondaryFs;
/** IGFS mode. */
private IgfsMode dfltMode = DFLT_MODE;
@@ -517,7 +518,7 @@ public class IgfsConfiguration {
*
* @return Secondary file system.
*/
- public Igfs getSecondaryFileSystem() {
+ public IgfsSecondaryFileSystem getSecondaryFileSystem() {
return secondaryFs;
}
@@ -527,7 +528,7 @@ public class IgfsConfiguration {
*
* @param fileSystem
*/
- public void setSecondaryFileSystem(Igfs fileSystem) {
+ public void setSecondaryFileSystem(IgfsSecondaryFileSystem fileSystem) {
secondaryFs = fileSystem;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java b/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
deleted file mode 100644
index 48b9b58..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/Igfs.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import org.apache.ignite.*;
-import org.jetbrains.annotations.*;
-
-import java.io.*;
-import java.util.*;
-
-/**
- * Common file system interface. It provides a typical generalized "view" of any file system:
- * <ul>
- * <li>list directories or get information for a single path</li>
- * <li>create/move/delete files or directories</li>
- * <li>write/read data streams into/from files</li>
- * </ul>
- *
- * This is the minimum of functionality that is needed to work as secondary file system in dual modes of IGFS.
- */
-public interface Igfs {
- /** File property: user name. */
- public static final String PROP_USER_NAME = "usrName";
-
- /** File property: group name. */
- public static final String PROP_GROUP_NAME = "grpName";
-
- /** File property: permission. */
- public static final String PROP_PERMISSION = "permission";
-
- /**
- * Checks if the specified path exists in the file system.
- *
- * @param path Path to check for existence in the file system.
- * @return {@code True} if such file exists, otherwise - {@code false}.
- * @throws IgniteException In case of error.
- */
- public boolean exists(IgfsPath path);
-
- /**
- * Updates file information for the specified path. Existent properties, not listed in the passed collection,
- * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
- * will be removed from the stored properties or ignored if they don't exist in the file info.
- * <p>
- * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
- * to the secondary file system:
- * <ul>
- * <li>{@code usrName} - file owner name;</li>
- * <li>{@code grpName} - file owner group;</li>
- * <li>{@code permission} - Unix-style string representing file permissions.</li>
- * </ul>
- *
- * @param path File path to set properties for.
- * @param props Properties to update.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
-
- /**
- * Renames/moves a file.
- * <p>
- * You are free to rename/move data files as you wish, but directories can be only renamed.
- * You cannot move the directory between different parent directories.
- * <p>
- * Examples:
- * <ul>
- * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
- * <li>"/work" => "/work-2012.bkp"</li>
- * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
- * </ul>
- *
- * @param src Source file path to rename.
- * @param dest Destination file path. If destination path is a directory, then source file will be placed
- * into destination directory with original name.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If source file doesn't exist.
- */
- public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
-
- /**
- * Deletes file.
- *
- * @param path File path to delete.
- * @param recursive Delete non-empty directories recursively.
- * @return {@code True} in case of success, {@code false} otherwise.
- * @throws IgniteException In case of error.
- */
- public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
-
- /**
- * Creates directories under specified path.
- *
- * @param path Path of directories chain to create.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path) throws IgniteException;
-
- /**
- * Creates directories under specified path with the specified properties.
- *
- * @param path Path of directories chain to create.
- * @param props Metadata properties to set on created directories.
- * @throws IgniteException In case of error.
- */
- public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Lists file paths under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
-
- /**
- * Lists files under the specified path.
- *
- * @param path Path to list files under.
- * @return List of files under the specified path.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
-
- /**
- * Opens a file for reading.
- *
- * @param path File path to read.
- * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
- * @return File input stream to read data from.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist.
- */
- public IgfsReader open(IgfsPath path, int bufSize) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
-
- /**
- * Creates a file and opens it for writing.
- *
- * @param path File path to create.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
- * @param replication Replication factor.
- * @param blockSize Block size.
- * @param props File properties to set.
- * @return File output stream to write data to.
- * @throws IgniteException In case of error.
- */
- public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize,
- @Nullable Map<String, String> props) throws IgniteException;
-
- /**
- * Opens an output stream to an existing file for appending data.
- *
- * @param path File path to append.
- * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
- * @param create Create file if it doesn't exist yet.
- * @param props File properties to set only in case it file was just created.
- * @return File output stream to append data to.
- * @throws IgniteException In case of error.
- * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
- */
- public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
- throws IgniteException;
-
- /**
- * Gets file information for the specified path.
- *
- * @param path Path to get information for.
- * @return File information for specified path or {@code null} if such path does not exist.
- * @throws IgniteException In case of error.
- */
- public IgfsFile info(IgfsPath path) throws IgniteException;
-
- /**
- * Gets used space in bytes.
- *
- * @return Used space in bytes.
- * @throws IgniteException In case of error.
- */
- public long usedSpaceSize() throws IgniteException;
-
- /**
- * Gets the implementation specific properties of file system.
- *
- * @return Map of properties.
- */
- public Map<String,String> properties();
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
index c4f28c6..afd0314 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsBlockLocation.java
@@ -22,7 +22,7 @@ import java.util.*;
/**
* {@code IGFS} file's data block location in the grid. It is used to determine
* node affinity of a certain file block within the Grid by calling
- * {@link org.apache.ignite.IgniteFs#affinity(IgfsPath, long, long)} method.
+ * {@link org.apache.ignite.IgniteFileSystem#affinity(IgfsPath, long, long)} method.
*/
public interface IgfsBlockLocation {
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
index 172dca1..550679a 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsFile.java
@@ -77,7 +77,7 @@ public interface IgfsFile {
/**
* Gets file last access time. File last access time is not updated automatically due to
* performance considerations and can be updated on demand with
- * {@link org.apache.ignite.IgniteFs#setTimes(IgfsPath, long, long)} method.
+ * {@link org.apache.ignite.IgniteFileSystem#setTimes(IgfsPath, long, long)} method.
* <p>
* By default last access time equals file creation time.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
index 308dbcb..746aa6f 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
@@ -17,13 +17,15 @@
package org.apache.ignite.igfs;
+import org.apache.ignite.igfs.secondary.*;
+
import java.io.*;
/**
* {@code IGFS} input stream to read data from the file system.
* It provides several additional methods for asynchronous access.
*/
-public abstract class IgfsInputStream extends InputStream implements IgfsReader {
+public abstract class IgfsInputStream extends InputStream implements IgfsSecondaryFileSystemReader {
/**
* Gets file length during file open.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
deleted file mode 100644
index fff8ca3..0000000
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsReader.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.igfs;
-
-import java.io.*;
-
-/**
- * The simplest data input interface to read from secondary file system in dual modes.
- */
-public interface IgfsReader extends Closeable {
- /**
- * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
- * read.
- *
- * @param pos Position in the input stream to seek.
- * @param buf Buffer into which data is read.
- * @param off Offset in the buffer from which stream data should be written.
- * @param len The number of bytes to read.
- * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
- * @throws IOException In case of any exception.
- */
- public int read(long pos, byte[] buf, int off, int len) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
index bf2636f..d7fd866 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsInputStreamJobAdapter.java
@@ -32,7 +32,7 @@ import java.io.*;
*/
public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter {
/** {@inheritDoc} */
- @Override public final Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in)
+ @Override public final Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in)
throws IgniteException, IOException {
in.seek(range.start());
@@ -48,5 +48,5 @@ public abstract class IgfsInputStreamJobAdapter extends IgfsJobAdapter {
* @throws IgniteException If execution failed.
* @throws IOException If IO exception encountered while working with stream.
*/
- public abstract Object execute(IgniteFs igfs, IgfsRangeInputStream in) throws IgniteException, IOException;
+ public abstract Object execute(IgniteFileSystem igfs, IgfsRangeInputStream in) throws IgniteException, IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
index d12fc79..0fdc0f2 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsJob.java
@@ -26,7 +26,7 @@ import java.io.*;
* Defines executable unit for {@link IgfsTask}. Before this job is executed, it is assigned one of the
* ranges provided by the {@link IgfsRecordResolver} passed to one of the {@code IgniteFs.execute(...)} methods.
* <p>
- * {@link #execute(org.apache.ignite.IgniteFs, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this
+ * {@link #execute(org.apache.ignite.IgniteFileSystem, IgfsFileRange, org.apache.ignite.igfs.IgfsInputStream)} method is given {@link IgfsFileRange} this
* job is expected to operate on, and already opened {@link org.apache.ignite.igfs.IgfsInputStream} for the file this range belongs to.
* <p>
* Note that provided input stream has position already adjusted to range start. However, it will not
@@ -52,7 +52,7 @@ public interface IgfsJob {
* @throws IgniteException If execution failed.
* @throws IOException If file system operation resulted in IO exception.
*/
- public Object execute(IgniteFs igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException,
+ public Object execute(IgniteFileSystem igfs, IgfsFileRange range, IgfsInputStream in) throws IgniteException,
IOException;
/**
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
index 52afeda..cc1d73f 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsRecordResolver.java
@@ -51,6 +51,6 @@ public interface IgfsRecordResolver extends Serializable {
* @throws IgniteException If resolve failed.
* @throws IOException If resolve failed.
*/
- @Nullable public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Nullable public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord) throws IgniteException, IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
index e42b012..e257c38 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/IgfsTask.java
@@ -88,7 +88,7 @@ public abstract class IgfsTask<T, R> extends ComputeTaskAdapter<IgfsTaskArgs<T>,
assert ignite != null;
assert args != null;
- IgniteFs fs = ignite.fileSystem(args.igfsName());
+ IgniteFileSystem fs = ignite.fileSystem(args.igfsName());
IgfsProcessorAdapter igfsProc = ((IgniteKernal) ignite).context().igfs();
Map<ComputeJob, ClusterNode> splitMap = new HashMap<>();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
index c1c15a5..c9ed821 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsByteDelimiterRecordResolver.java
@@ -20,7 +20,6 @@ package org.apache.ignite.igfs.mapreduce.records;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.tostring.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
@@ -78,7 +77,7 @@ public class IgfsByteDelimiterRecordResolver implements IgfsRecordResolver, Exte
}
/** {@inheritDoc} */
- @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord) throws IgniteException, IOException {
long suggestedStart = suggestedRecord.start();
long suggestedEnd = suggestedStart + suggestedRecord.length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
index fe22627..bcc8f69 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/mapreduce/records/IgfsFixedLengthRecordResolver.java
@@ -52,7 +52,7 @@ public class IgfsFixedLengthRecordResolver implements IgfsRecordResolver, Extern
}
/** {@inheritDoc} */
- @Override public IgfsFileRange resolveRecords(IgniteFs fs, IgfsInputStream stream,
+ @Override public IgfsFileRange resolveRecords(IgniteFileSystem fs, IgfsInputStream stream,
IgfsFileRange suggestedRecord)
throws IgniteException, IOException {
long suggestedEnd = suggestedRecord.start() + suggestedRecord.length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
new file mode 100644
index 0000000..88fcae4
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.secondary;
+
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Common file system interface. It provides a typical generalized "view" of any file system:
+ * <ul>
+ * <li>list directories or get information for a single path</li>
+ * <li>create/move/delete files or directories</li>
+ * <li>write/read data streams into/from files</li>
+ * </ul>
+ *
+ * This is the minimum of functionality that is needed to work as secondary file system in dual modes of IGFS.
+ */
+public interface IgfsSecondaryFileSystem {
+ /** File property: user name. */
+ public static final String PROP_USER_NAME = "usrName";
+
+ /** File property: group name. */
+ public static final String PROP_GROUP_NAME = "grpName";
+
+ /** File property: permission. */
+ public static final String PROP_PERMISSION = "permission";
+
+ /**
+ * Checks if the specified path exists in the file system.
+ *
+ * @param path Path to check for existence in the file system.
+ * @return {@code True} if such file exists, otherwise - {@code false}.
+ * @throws IgniteException In case of error.
+ */
+ public boolean exists(IgfsPath path);
+
+ /**
+ * Updates file information for the specified path. Existent properties, not listed in the passed collection,
+ * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
+ * will be removed from the stored properties or ignored if they don't exist in the file info.
+ * <p>
+ * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
+ * to the secondary file system:
+ * <ul>
+ * <li>{@code usrName} - file owner name;</li>
+ * <li>{@code grpName} - file owner group;</li>
+ * <li>{@code permission} - Unix-style string representing file permissions.</li>
+ * </ul>
+ *
+ * @param path File path to set properties for.
+ * @param props Properties to update.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
+
+ /**
+ * Renames/moves a file.
+ * <p>
+ * You are free to rename/move data files as you wish, but directories can be only renamed.
+ * You cannot move the directory between different parent directories.
+ * <p>
+ * Examples:
+ * <ul>
+ * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
+ * <li>"/work" => "/work-2012.bkp"</li>
+ * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
+ * </ul>
+ *
+ * @param src Source file path to rename.
+ * @param dest Destination file path. If destination path is a directory, then source file will be placed
+ * into destination directory with original name.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If source file doesn't exist.
+ */
+ public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
+
+ /**
+ * Deletes file.
+ *
+ * @param path File path to delete.
+ * @param recursive Delete non-empty directories recursively.
+ * @return {@code True} in case of success, {@code false} otherwise.
+ * @throws IgniteException In case of error.
+ */
+ public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
+
+ /**
+ * Creates directories under specified path.
+ *
+ * @param path Path of directories chain to create.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path) throws IgniteException;
+
+ /**
+ * Creates directories under specified path with the specified properties.
+ *
+ * @param path Path of directories chain to create.
+ * @param props Metadata properties to set on created directories.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Lists file paths under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
+
+ /**
+ * Lists files under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
+
+ /**
+ * Opens a file for reading.
+ *
+ * @param path File path to read.
+ * @param bufSize Read buffer size (bytes) or {@code zero} to use default value.
+ * @return File input stream to read data from.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public IgfsSecondaryFileSystemReader open(IgfsPath path, int bufSize) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
+
+ /**
+ * Creates a file and opens it for writing.
+ *
+ * @param path File path to create.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param overwrite Overwrite file if it already exists. Note: you cannot overwrite an existent directory.
+ * @param replication Replication factor.
+ * @param blockSize Block size.
+ * @param props File properties to set.
+ * @return File output stream to write data to.
+ * @throws IgniteException In case of error.
+ */
+ public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication, long blockSize,
+ @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Opens an output stream to an existing file for appending data.
+ *
+ * @param path File path to append.
+ * @param bufSize Write buffer size (bytes) or {@code zero} to use default value.
+ * @param create Create file if it doesn't exist yet.
+ * @param props File properties to set only in case it file was just created.
+ * @return File output stream to append data to.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
+ */
+ public OutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
+ throws IgniteException;
+
+ /**
+ * Gets file information for the specified path.
+ *
+ * @param path Path to get information for.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile info(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets used space in bytes.
+ *
+ * @return Used space in bytes.
+ * @throws IgniteException In case of error.
+ */
+ public long usedSpaceSize() throws IgniteException;
+
+ /**
+ * Gets the implementation specific properties of file system.
+ *
+ * @return Map of properties.
+ */
+ public Map<String,String> properties();
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
new file mode 100644
index 0000000..02a135e
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystemReader.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.igfs.secondary;
+
+import java.io.*;
+
+/**
+ * The simplest data input interface to read from secondary file system in dual modes.
+ */
+public interface IgfsSecondaryFileSystemReader extends Closeable {
+ /**
+ * Read up to the specified number of bytes, from a given position within a file, and return the number of bytes
+ * read.
+ *
+ * @param pos Position in the input stream to seek.
+ * @param buf Buffer into which data is read.
+ * @param off Offset in the buffer from which stream data should be written.
+ * @param len The number of bytes to read.
+ * @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
+ * @throws IOException In case of any exception.
+ */
+ public int read(long pos, byte[] buf, int off, int len) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
new file mode 100644
index 0000000..8bd668d
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/package.html
@@ -0,0 +1,24 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+<body>
+ <!-- Package description. -->
+ Contains APIs for IGFS secondary file system.
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
index 7557efe..3ede8d5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteEx.java
@@ -117,7 +117,7 @@ public interface IgniteEx extends Ignite {
* @param name IGFS name.
* @return IGFS.
*/
- @Nullable public IgniteFs igfsx(@Nullable String name);
+ @Nullable public IgniteFileSystem igfsx(@Nullable String name);
/**
* Get Hadoop facade.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
index 16ffc30..2a371ac 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/IgniteKernal.java
@@ -2358,11 +2358,11 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Override public IgniteFs fileSystem(String name) {
+ @Override public IgniteFileSystem fileSystem(String name) {
guard();
try{
- IgniteFs fs = ctx.igfs().igfs(name);
+ IgniteFileSystem fs = ctx.igfs().igfs(name);
if (fs == null)
throw new IllegalArgumentException("IGFS is not configured: " + name);
@@ -2375,7 +2375,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Nullable @Override public IgniteFs igfsx(@Nullable String name) {
+ @Nullable @Override public IgniteFileSystem igfsx(@Nullable String name) {
guard();
try {
@@ -2387,7 +2387,7 @@ public class IgniteKernal implements IgniteEx, IgniteMXBean, Externalizable {
}
/** {@inheritDoc} */
- @Override public Collection<IgniteFs> fileSystems() {
+ @Override public Collection<IgniteFileSystem> fileSystems() {
guard();
try {
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/dd06414a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
index 1479a88..cb8b833 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
@@ -32,7 +32,7 @@ import java.util.*;
/**
* Igfs supporting asynchronous operations.
*/
-public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFs> implements IgfsEx {
+public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFileSystem> implements IgfsEx {
/** */
private final IgfsImpl igfs;
[6/6] incubator-ignite git commit: # IGNITE-386: Interface split.
Posted by vo...@apache.org.
# IGNITE-386: Interface split.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/d83c88af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/d83c88af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/d83c88af
Branch: refs/heads/ignite-386
Commit: d83c88af03901a315f27b76caa47b0bb3c362ef5
Parents: 2cbc7ea
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Wed Mar 4 12:26:50 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Wed Mar 4 12:26:50 2015 +0300
----------------------------------------------------------------------
.../org/apache/ignite/IgniteFileSystem.java | 127 ++++-
.../org/apache/ignite/igfs/IgfsInputStream.java | 4 +-
.../internal/processors/igfs/IgfsAsyncImpl.java | 5 +-
.../ignite/internal/processors/igfs/IgfsEx.java | 8 +
.../internal/processors/igfs/IgfsImpl.java | 25 +-
.../processors/igfs/IgfsInputStreamAdapter.java | 4 +-
.../igfs/IgfsSecondaryFileSystemImpl.java | 121 +++++
...heIgfsPerBlockLruEvictionPolicySelfTest.java | 485 -------------------
.../processors/igfs/IgfsAbstractSelfTest.java | 66 +--
...sCachePerBlockLruEvictionPolicySelfTest.java | 485 +++++++++++++++++++
.../igfs/IgfsDataManagerSelfTest.java | 1 -
.../igfs/IgfsDualAbstractSelfTest.java | 10 +-
.../processors/igfs/IgfsMetricsSelfTest.java | 6 +-
.../processors/igfs/IgfsModesSelfTest.java | 2 +-
.../ignite/testsuites/IgniteIgfsTestSuite.java | 2 +-
.../igfs/HadoopIgfsDualAbstractSelfTest.java | 2 +-
.../HadoopDefaultMapReducePlannerSelfTest.java | 11 +-
17 files changed, 814 insertions(+), 550 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
index 280628d..7067faf 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -20,7 +20,6 @@ package org.apache.ignite;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
-import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.lang.*;
import org.jetbrains.annotations.*;
@@ -57,7 +56,7 @@ import java.util.*;
* <p>
* <b>NOTE:</b> integration with Hadoop is available only in {@code In-Memory Accelerator For Hadoop} edition.
*/
-public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSupport {
+public interface IgniteFileSystem extends IgniteAsyncSupport {
/** IGFS scheme name. */
public static final String IGFS_SCHEME = "igfs";
@@ -105,7 +104,7 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
* @throws IgniteException In case of error.
* @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist.
*/
- @Override public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
+ public IgfsInputStream open(IgfsPath path, int bufSize) throws IgniteException;
/**
* Opens a file for reading.
@@ -127,7 +126,7 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
* @return File output stream to write data to.
* @throws IgniteException In case of error.
*/
- @Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
+ public IgfsOutputStream create(IgfsPath path, boolean overwrite) throws IgniteException;
/**
* Creates a file and opens it for writing.
@@ -141,7 +140,7 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
* @return File output stream to write data to.
* @throws IgniteException In case of error.
*/
- @Override public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ public IgfsOutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
long blockSize, @Nullable Map<String, String> props) throws IgniteException;
/**
@@ -184,8 +183,8 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
* @throws IgniteException In case of error.
* @throws org.apache.ignite.igfs.IgfsFileNotFoundException If path doesn't exist and create flag is {@code false}.
*/
- @Override public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create,
- @Nullable Map<String, String> props) throws IgniteException;
+ public IgfsOutputStream append(IgfsPath path, int bufSize, boolean create, @Nullable Map<String, String> props)
+ throws IgniteException;
/**
* Sets last access time and last modification time for a given path. If argument is {@code null},
@@ -339,6 +338,120 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
@Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
long maxRangeLen, @Nullable T arg) throws IgniteException;
+ /**
+ * Checks if the specified path exists in the file system.
+ *
+ * @param path Path to check for existence in the file system.
+ * @return {@code True} if such file exists, otherwise - {@code false}.
+ * @throws IgniteException In case of error.
+ */
+ public boolean exists(IgfsPath path);
+
+ /**
+ * Updates file information for the specified path. Existent properties, not listed in the passed collection,
+ * will not be affected. Other properties will be added or overwritten. Passed properties with {@code null} values
+ * will be removed from the stored properties or ignored if they don't exist in the file info.
+ * <p>
+ * When working in {@code DUAL_SYNC} or {@code DUAL_ASYNC} modes only the following properties will be propagated
+ * to the secondary file system:
+ * <ul>
+ * <li>{@code usrName} - file owner name;</li>
+ * <li>{@code grpName} - file owner group;</li>
+ * <li>{@code permission} - Unix-style string representing file permissions.</li>
+ * </ul>
+ *
+ * @param path File path to set properties for.
+ * @param props Properties to update.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException;
+
+ /**
+ * Renames/moves a file.
+ * <p>
+ * You are free to rename/move data files as you wish, but directories can be only renamed.
+ * You cannot move the directory between different parent directories.
+ * <p>
+ * Examples:
+ * <ul>
+ * <li>"/work/file.txt" => "/home/project/Presentation Scenario.txt"</li>
+ * <li>"/work" => "/work-2012.bkp"</li>
+ * <li>"/work" => "<strike>/backups/work</strike>" - such operation is restricted for directories.</li>
+ * </ul>
+ *
+ * @param src Source file path to rename.
+ * @param dest Destination file path. If destination path is a directory, then source file will be placed
+ * into destination directory with original name.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If source file doesn't exist.
+ */
+ public void rename(IgfsPath src, IgfsPath dest) throws IgniteException;
+
+ /**
+ * Deletes file.
+ *
+ * @param path File path to delete.
+ * @param recursive Delete non-empty directories recursively.
+ * @return {@code True} in case of success, {@code false} otherwise.
+ * @throws IgniteException In case of error.
+ */
+ public boolean delete(IgfsPath path, boolean recursive) throws IgniteException;
+
+ /**
+ * Creates directories under specified path.
+ *
+ * @param path Path of directories chain to create.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path) throws IgniteException;
+
+ /**
+ * Creates directories under specified path with the specified properties.
+ *
+ * @param path Path of directories chain to create.
+ * @param props Metadata properties to set on created directories.
+ * @throws IgniteException In case of error.
+ */
+ public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException;
+
+ /**
+ * Lists file paths under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException;
+
+ /**
+ * Lists files under the specified path.
+ *
+ * @param path Path to list files under.
+ * @return List of files under the specified path.
+ * @throws IgniteException In case of error.
+ * @throws IgfsFileNotFoundException If path doesn't exist.
+ */
+ public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets file information for the specified path.
+ *
+ * @param path Path to get information for.
+ * @return File information for specified path or {@code null} if such path does not exist.
+ * @throws IgniteException In case of error.
+ */
+ public IgfsFile info(IgfsPath path) throws IgniteException;
+
+ /**
+ * Gets used space in bytes.
+ *
+ * @return Used space in bytes.
+ * @throws IgniteException In case of error.
+ */
+ public long usedSpaceSize() throws IgniteException;
+
/** {@inheritDoc} */
@Override public IgniteFileSystem withAsync();
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
index 79d69a8..c2ddbb0 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/IgfsInputStream.java
@@ -25,7 +25,7 @@ import java.io.*;
* {@code IGFS} input stream to read data from the file system.
* It provides several additional methods for asynchronous access.
*/
-public abstract class IgfsInputStream extends InputStream implements IgfsSecondaryFileSystemPositionedReadable {
+public abstract class IgfsInputStream extends InputStream {
/**
* Gets file length during file open.
*
@@ -78,5 +78,5 @@ public abstract class IgfsInputStream extends InputStream implements IgfsSeconda
* @return Total number of bytes read into the buffer, or -1 if there is no more data (EOF).
* @throws IOException In case of IO exception.
*/
- @Override public abstract int read(long pos, byte[] buf, int off, int len) throws IOException;
+ public abstract int read(long pos, byte[] buf, int off, int len) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
index cb8b833..7aeeb46 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsAsyncImpl.java
@@ -21,6 +21,7 @@ import org.apache.ignite.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.lang.*;
@@ -310,7 +311,7 @@ public class IgfsAsyncImpl extends AsyncSupportAdapter<IgniteFileSystem> impleme
}
/** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return igfs.properties();
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return igfs.asSecondary();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index 7e96c19..492218f 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.*;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.lang.*;
import org.jetbrains.annotations.*;
@@ -152,4 +153,11 @@ public interface IgfsEx extends IgniteFileSystem {
* @return {@code True} if proxy.
*/
public boolean isProxy(URI path);
+
+ /**
+ * Return the given IGFS as a secondary file system.
+ *
+ * @return Secondary file system wrapper.
+ */
+ public IgfsSecondaryFileSystem asSecondary();
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
index c3c81f9..f3fcc83 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsImpl.java
@@ -125,6 +125,7 @@ public final class IgfsImpl implements IgfsEx {
* @param igfsCtx Context.
* @throws IgniteCheckedException In case of error.
*/
+ @SuppressWarnings("ConstantConditions")
IgfsImpl(IgfsContext igfsCtx) throws IgniteCheckedException {
assert igfsCtx != null;
@@ -939,6 +940,7 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
+ @SuppressWarnings("unchecked")
@Override public Collection<IgfsPath> listPaths(final IgfsPath path) {
if (enterBusy()) {
try {
@@ -1068,11 +1070,6 @@ public final class IgfsImpl implements IgfsEx {
}
/** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
-
- /** {@inheritDoc} */
@Override public IgfsInputStreamAdapter open(IgfsPath path) {
return open(path, cfg.getStreamBufferSize(), cfg.getSequentialReadsBeforePrefetch());
}
@@ -1194,7 +1191,7 @@ public final class IgfsImpl implements IgfsEx {
IgfsMode mode = modeRslvr.resolveMode(path);
- IgfsFileWorkerBatch batch = null;
+ IgfsFileWorkerBatch batch;
if (mode == PROXY)
throw new IgniteException("PROXY mode cannot be used in IGFS directly: " + path);
@@ -1251,6 +1248,8 @@ public final class IgfsImpl implements IgfsEx {
IgfsFileInfo oldInfo = meta.info(oldId);
+ assert oldInfo != null;
+
if (oldInfo.isDirectory())
throw new IgfsPathAlreadyExistsException("Failed to create file (path points to a " +
"directory): " + path);
@@ -1269,7 +1268,7 @@ public final class IgfsImpl implements IgfsEx {
info = meta.lock(info.id());
IgfsEventAwareOutputStream os = new IgfsEventAwareOutputStream(path, info, parentId,
- bufSize == 0 ? cfg.getStreamBufferSize() : bufSize, mode, batch);
+ bufSize == 0 ? cfg.getStreamBufferSize() : bufSize, mode, null);
if (evts.isRecordable(EVT_IGFS_FILE_OPENED_WRITE))
evts.record(new IgfsEvent(path, localNode(), EVT_IGFS_FILE_OPENED_WRITE));
@@ -1340,8 +1339,7 @@ public final class IgfsImpl implements IgfsEx {
if (parentId == null)
throw new IgfsInvalidPathException("Failed to resolve parent directory: " + path);
- info = new IgfsFileInfo(cfg.getBlockSize(), /**affinity key*/null, evictExclude(path,
- mode == PRIMARY), props);
+ info = new IgfsFileInfo(cfg.getBlockSize(), /**affinity key*/null, evictExclude(path, true), props);
IgniteUuid oldId = meta.putIfAbsent(parentId, path.name(), info);
@@ -1352,6 +1350,8 @@ public final class IgfsImpl implements IgfsEx {
evts.record(new IgfsEvent(path, localNode(), EVT_IGFS_FILE_CREATED));
}
+ assert info != null;
+
if (!info.isFile())
throw new IgfsInvalidPathException("Failed to open file (not a file): " + path);
@@ -1834,6 +1834,7 @@ public final class IgfsImpl implements IgfsEx {
* @param arg Optional task argument.
* @return Execution future.
*/
+ @SuppressWarnings("unchecked")
<T, R> IgniteInternalFuture<R> executeAsync(Class<? extends IgfsTask<T, R>> taskCls,
@Nullable IgfsRecordResolver rslvr, Collection<IgfsPath> paths, boolean skipNonExistentFiles,
long maxRangeLen, @Nullable T arg) {
@@ -2130,6 +2131,7 @@ public final class IgfsImpl implements IgfsEx {
*/
private class FormatMessageListener implements GridMessageListener {
/** {@inheritDoc} */
+ @SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@Override public void onMessage(UUID nodeId, Object msg) {
if (msg instanceof IgfsDeleteMessage) {
ClusterNode node = igfsCtx.kernalContext().discovery().node(nodeId);
@@ -2220,4 +2222,9 @@ public final class IgfsImpl implements IgfsEx {
return mode == PROXY;
}
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return new IgfsSecondaryFileSystemImpl(this);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
index 51e57db..fc333b5 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsInputStreamAdapter.java
@@ -18,13 +18,15 @@
package org.apache.ignite.internal.processors.igfs;
import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
import java.io.*;
/**
* Implementation adapter providing necessary methods.
*/
-public abstract class IgfsInputStreamAdapter extends IgfsInputStream {
+public abstract class IgfsInputStreamAdapter extends IgfsInputStream
+ implements IgfsSecondaryFileSystemPositionedReadable {
/** {@inheritDoc} */
@Override public long length() {
return fileInfo().length();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
new file mode 100644
index 0000000..683b317
--- /dev/null
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsSecondaryFileSystemImpl.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.igfs.secondary.*;
+import org.jetbrains.annotations.*;
+
+import java.io.*;
+import java.util.*;
+
+/**
+ * Secondary file system over native IGFS.
+ */
+class IgfsSecondaryFileSystemImpl implements IgfsSecondaryFileSystem {
+ /** Delegate. */
+ private final IgfsImpl igfs;
+
+ /**
+ * Constructor.
+ *
+ * @param igfs Delegate.
+ */
+ IgfsSecondaryFileSystemImpl(IgfsImpl igfs) {
+ this.igfs = igfs;
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean exists(IgfsPath path) {
+ return igfs.exists(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile update(IgfsPath path, Map<String, String> props) throws IgniteException {
+ return igfs.update(path, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void rename(IgfsPath src, IgfsPath dest) throws IgniteException {
+ igfs.rename(src, dest);
+ }
+
+ /** {@inheritDoc} */
+ @Override public boolean delete(IgfsPath path, boolean recursive) throws IgniteException {
+ return igfs.delete(path, recursive);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path) throws IgniteException {
+ igfs.mkdirs(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public void mkdirs(IgfsPath path, @Nullable Map<String, String> props) throws IgniteException {
+ igfs.mkdirs(path, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsPath> listPaths(IgfsPath path) throws IgniteException {
+ return igfs.listPaths(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public Collection<IgfsFile> listFiles(IgfsPath path) throws IgniteException {
+ return igfs.listFiles(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystemPositionedReadable open(IgfsPath path, int bufSize)
+ throws IgniteException {
+ return igfs.open(path, bufSize);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, boolean overwrite) throws IgniteException {
+ return igfs.create(path, overwrite);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream create(IgfsPath path, int bufSize, boolean overwrite, int replication,
+ long blockSize, @Nullable Map<String, String> props) throws IgniteException {
+ return igfs.create(path, bufSize, overwrite, replication, blockSize, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public OutputStream append(IgfsPath path, int bufSize, boolean create,
+ @Nullable Map<String, String> props) throws IgniteException {
+ return igfs.append(path, bufSize, create, props);
+ }
+
+ /** {@inheritDoc} */
+ @Override public IgfsFile info(IgfsPath path) throws IgniteException {
+ return igfs.info(path);
+ }
+
+ /** {@inheritDoc} */
+ @Override public long usedSpaceSize() throws IgniteException {
+ return igfs.usedSpaceSize();
+ }
+
+ /** {@inheritDoc} */
+ @Override public Map<String, String> properties() {
+ return Collections.emptyMap();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
deleted file mode 100644
index a2e1c81..0000000
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/GridCacheIgfsPerBlockLruEvictionPolicySelfTest.java
+++ /dev/null
@@ -1,485 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.internal.processors.igfs;
-
-import org.apache.ignite.*;
-import org.apache.ignite.cache.*;
-import org.apache.ignite.cache.eviction.igfs.*;
-import org.apache.ignite.configuration.*;
-import org.apache.ignite.igfs.*;
-import org.apache.ignite.internal.*;
-import org.apache.ignite.internal.processors.cache.*;
-import org.apache.ignite.internal.util.lang.*;
-import org.apache.ignite.internal.util.typedef.*;
-import org.apache.ignite.internal.util.typedef.internal.*;
-import org.apache.ignite.spi.discovery.tcp.*;
-import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
-import org.apache.ignite.testframework.*;
-
-import java.util.*;
-import java.util.concurrent.*;
-
-import static org.apache.ignite.cache.CacheAtomicityMode.*;
-import static org.apache.ignite.cache.CacheMode.*;
-import static org.apache.ignite.igfs.IgfsMode.*;
-
-/**
- * Tests for IGFS per-block LR eviction policy.
- */
-@SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
-public class GridCacheIgfsPerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstractTest {
- /** Primary IGFS name. */
- private static final String IGFS_PRIMARY = "igfs-primary";
-
- /** Primary IGFS name. */
- private static final String IGFS_SECONDARY = "igfs-secondary";
-
- /** Secondary file system REST endpoint configuration map. */
- private static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
- put("type", "tcp");
- put("port", "11500");
- }};
-
- /** File working in PRIMARY mode. */
- public static final IgfsPath FILE = new IgfsPath("/file");
-
- /** File working in DUAL mode. */
- public static final IgfsPath FILE_RMT = new IgfsPath("/fileRemote");
-
- /** Primary IGFS instances. */
- private static IgfsImpl igfsPrimary;
-
- /** Secondary IGFS instance. */
- private static IgniteFileSystem secondaryFs;
-
- /** Primary file system data cache. */
- private static GridCacheAdapter<IgfsBlockKey, byte[]> dataCache;
-
- /** Eviction policy */
- private static CacheIgfsPerBlockLruEvictionPolicy evictPlc;
-
- /**
- * Start a grid with the primary file system.
- *
- * @throws Exception If failed.
- */
- private void startPrimary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(IGFS_PRIMARY);
- igfsCfg.setBlockSize(512);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setPrefetchBlocks(1);
- igfsCfg.setSequentialReadsBeforePrefetch(Integer.MAX_VALUE);
- igfsCfg.setSecondaryFileSystem(secondaryFs);
-
- Map<String, IgfsMode> pathModes = new HashMap<>();
-
- pathModes.put(FILE_RMT.toString(), DUAL_SYNC);
-
- igfsCfg.setPathModes(pathModes);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- evictPlc = new CacheIgfsPerBlockLruEvictionPolicy();
-
- dataCacheCfg.setEvictionPolicy(evictPlc);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid-primary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- Ignite g = G.start(cfg);
-
- igfsPrimary = (IgfsImpl)g.fileSystem(IGFS_PRIMARY);
-
- dataCache = igfsPrimary.context().kernalContext().cache().internalCache(
- igfsPrimary.context().configuration().getDataCacheName());
- }
-
- /**
- * Start a grid with the secondary file system.
- *
- * @throws Exception If failed.
- */
- private void startSecondary() throws Exception {
- IgfsConfiguration igfsCfg = new IgfsConfiguration();
-
- igfsCfg.setDataCacheName("dataCache");
- igfsCfg.setMetaCacheName("metaCache");
- igfsCfg.setName(IGFS_SECONDARY);
- igfsCfg.setBlockSize(512);
- igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
-
- CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
-
- dataCacheCfg.setName("dataCache");
- dataCacheCfg.setCacheMode(PARTITIONED);
- dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
- dataCacheCfg.setBackups(0);
- dataCacheCfg.setQueryIndexEnabled(false);
- dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
-
- metaCacheCfg.setName("metaCache");
- metaCacheCfg.setCacheMode(REPLICATED);
- metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
- metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
- metaCacheCfg.setQueryIndexEnabled(false);
- metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
-
- IgniteConfiguration cfg = new IgniteConfiguration();
-
- cfg.setGridName("grid-secondary");
-
- TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
-
- discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
-
- cfg.setDiscoverySpi(discoSpi);
- cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
- cfg.setIgfsConfiguration(igfsCfg);
-
- cfg.setLocalHost("127.0.0.1");
- cfg.setConnectorConfiguration(null);
-
- Ignite g = G.start(cfg);
-
- secondaryFs = g.fileSystem(IGFS_SECONDARY);
- }
-
- /** {@inheritDoc} */
- @Override protected void afterTest() throws Exception {
- try {
- // Cleanup.
- igfsPrimary.format();
-
- while (!dataCache.isEmpty())
- U.sleep(100);
-
- checkEvictionPolicy(0, 0);
- }
- finally {
- stopAllGrids(false);
- }
- }
-
- /**
- * Startup primary and secondary file systems.
- *
- * @throws Exception If failed.
- */
- private void start() throws Exception {
- startSecondary();
- startPrimary();
-
- evictPlc.setMaxBlocks(0);
- evictPlc.setMaxSize(0);
- evictPlc.setExcludePaths(null);
- }
-
- /**
- * Test how evictions are handled for a file working in PRIMARY mode.
- *
- * @throws Exception If failed.
- */
- public void testFilePrimary() throws Exception {
- start();
-
- // Create file in primary mode. It must not be propagated to eviction policy.
- igfsPrimary.create(FILE, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE).blockSize();
-
- append(FILE, blockSize);
-
- checkEvictionPolicy(0, 0);
-
- read(FILE, 0, blockSize);
-
- checkEvictionPolicy(0, 0);
- }
-
- /**
- * Test how evictions are handled for a file working in PRIMARY mode.
- *
- * @throws Exception If failed.
- */
- public void testFileDual() throws Exception {
- start();
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- // File write.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(1, blockSize);
-
- // One more write.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(2, blockSize * 2);
-
- // Read.
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(2, blockSize * 2);
- }
-
- /**
- * Ensure that a DUAL mode file is not propagated to eviction policy
- *
- * @throws Exception If failed.
- */
- public void testFileDualExclusion() throws Exception {
- start();
-
- evictPlc.setExcludePaths(Collections.singleton(FILE_RMT.toString()));
-
- // Create file in primary mode. It must not be propagated to eviction policy.
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(0, 0);
-
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(0, 0);
- }
-
- /**
- * Ensure that exception is thrown in case we are trying to rename file with one exclude setting to the file with
- * another.
- *
- * @throws Exception If failed.
- */
- public void testRenameDifferentExcludeSettings() throws Exception {
- start();
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfsPrimary.rename(FILE, FILE_RMT);
-
- return null;
- }
- }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
- "(need to copy and remove)");
-
- GridTestUtils.assertThrows(log, new Callable<Object>() {
- @Override public Object call() throws Exception {
- igfsPrimary.rename(FILE_RMT, FILE);
-
- return null;
- }
- }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
- "(need to copy and remove)");
- }
-
- /**
- * Test eviction caused by too much blocks.
- *
- * @throws Exception If failed.
- */
- public void testBlockCountEviction() throws Exception {
- start();
-
- int blockCnt = 3;
-
- evictPlc.setMaxBlocks(blockCnt);
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- checkEvictionPolicy(0, 0);
-
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- // Write blocks up to the limit.
- append(FILE_RMT, blockSize * blockCnt);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Write one more block what should cause eviction.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the first block.
- read(FILE_RMT, 0, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
- checkMetrics(1, 1);
- }
-
- /**
- * Test eviction caused by too big data size.
- *
- * @throws Exception If failed.
- */
- public void testDataSizeEviction() throws Exception {
- start();
-
- igfsPrimary.create(FILE_RMT, true).close();
-
- int blockCnt = 3;
- int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
-
- evictPlc.setMaxSize(blockSize * blockCnt);
-
- // Write blocks up to the limit.
- append(FILE_RMT, blockSize * blockCnt);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Reset metrics.
- igfsPrimary.resetMetrics();
-
- // Read the first block what should cause reordering.
- read(FILE_RMT, 0, blockSize);
-
- checkMetrics(1, 0);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Write one more block what should cause eviction of the block 2.
- append(FILE_RMT, blockSize);
-
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the first block.
- read(FILE_RMT, 0, blockSize);
-
- checkMetrics(2, 0);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
-
- // Read the second block (which was evicted).
- read(FILE_RMT, blockSize, blockSize);
-
- checkMetrics(3, 1);
- checkEvictionPolicy(blockCnt, blockCnt * blockSize);
- }
-
- /**
- * Read some data from the given file with the given offset.
- *
- * @param path File path.
- * @param off Offset.
- * @param len Length.
- * @throws Exception If failed.
- */
- private void read(IgfsPath path, int off, int len) throws Exception {
- IgfsInputStream is = igfsPrimary.open(path);
-
- is.readFully(off, new byte[len]);
-
- is.close();
- }
-
- /**
- * Append some data to the given file.
- *
- * @param path File path.
- * @param len Data length.
- * @throws Exception If failed.
- */
- private void append(IgfsPath path, int len) throws Exception {
- IgfsOutputStream os = igfsPrimary.append(path, false);
-
- os.write(new byte[len]);
-
- os.close();
- }
-
- /**
- * Check metrics counters.
- *
- * @param blocksRead Expected blocks read.
- * @param blocksReadRmt Expected blocks read remote.
- * @throws Exception If failed.
- */
- public void checkMetrics(final long blocksRead, final long blocksReadRmt) throws Exception {
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- IgfsMetrics metrics = igfsPrimary.metrics();
-
- return metrics.blocksReadTotal() == blocksRead && metrics.blocksReadRemote() == blocksReadRmt;
- }
- }, 5000) : "Unexpected metrics [expectedBlocksReadTotal=" + blocksRead + ", actualBlocksReadTotal=" +
- igfsPrimary.metrics().blocksReadTotal() + ", expectedBlocksReadRemote=" + blocksReadRmt +
- ", actualBlocksReadRemote=" + igfsPrimary.metrics().blocksReadRemote() + ']';
- }
-
- /**
- * Check eviction policy state.
- *
- * @param curBlocks Current blocks.
- * @param curBytes Current bytes.
- */
- private void checkEvictionPolicy(final int curBlocks, final long curBytes) throws IgniteInterruptedCheckedException {
- assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
- @Override public boolean apply() {
- return evictPlc.getCurrentBlocks() == curBlocks && evictPlc.getCurrentSize() == curBytes;
- }
- }, 5000) : "Unexpected counts [expectedBlocks=" + curBlocks + ", actualBlocks=" + evictPlc.getCurrentBlocks() +
- ", expectedBytes=" + curBytes + ", currentBytes=" + curBytes + ']';
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 12ef0c1..31fc4a7 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -155,7 +155,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
igfsSecondary = (IgfsImpl) igniteSecondary.fileSystem("igfs-secondary");
- Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondary, PRIMARY_REST_CFG);
+ Ignite ignite = startGridWithIgfs("ignite", "igfs", mode, igfsSecondary.asSecondary(), PRIMARY_REST_CFG);
igfs = (IgfsImpl) ignite.fileSystem("igfs");
}
@@ -263,7 +263,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testExists() throws Exception {
- create(igfs, paths(DIR), null);
+ create(igfs.asSecondary(), paths(DIR), null);
checkExist(igfs, igfsSecondary, DIR);
}
@@ -374,7 +374,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
IgfsPath file1 = new IgfsPath("/file1");
IgfsPath file2 = new IgfsPath("/file2");
- create(igfs, null, paths(file1));
+ create(igfs.asSecondary(), null, paths(file1));
igfs.rename(file1, file2);
@@ -406,7 +406,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
IgfsPath dir1 = new IgfsPath("/dir1");
IgfsPath dir2 = new IgfsPath("/dir2");
- create(igfs, paths(dir1), null);
+ create(igfs.asSecondary(), paths(dir1), null);
igfs.rename(dir1, dir2);
@@ -435,7 +435,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testMoveFileDestinationRoot() throws Exception {
- create(igfs, paths(DIR, SUBDIR), paths(FILE));
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), paths(FILE));
igfs.rename(FILE, new IgfsPath());
@@ -527,7 +527,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testMoveDirectoryDestinationRoot() throws Exception {
- create(igfs, paths(DIR, SUBDIR, SUBSUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR, SUBSUBDIR), null);
igfs.rename(SUBSUBDIR, new IgfsPath());
@@ -543,7 +543,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testMoveDirectorySourceParentRoot() throws Exception {
IgfsPath dir = new IgfsPath("/" + SUBSUBDIR.name());
- create(igfs, paths(DIR_NEW, SUBDIR_NEW, dir), null);
+ create(igfs.asSecondary(), paths(DIR_NEW, SUBDIR_NEW, dir), null);
igfs.rename(dir, SUBDIR_NEW);
@@ -852,7 +852,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testOpen() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkFileContent(igfs, FILE, chunk);
}
@@ -887,9 +887,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testCreate() throws Exception {
- create(igfs, paths(DIR, SUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkFile(igfs, igfsSecondary, FILE, chunk);
}
@@ -902,7 +902,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testCreateParentRoot() throws Exception {
IgfsPath file = new IgfsPath("/" + FILE.name());
- createFile(igfs, file, true, chunk);
+ createFile(igfs.asSecondary(), file, true, chunk);
checkFile(igfs, igfsSecondary, file, chunk);
}
@@ -1093,7 +1093,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
assert igfs.exists(path);
}
- awaitFileClose(igfs, path);
+ awaitFileClose(igfs.asSecondary(), path);
checkFileContent(igfs, path, chunk);
}
@@ -1167,7 +1167,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
fut.get();
- awaitFileClose(igfs, FILE);
+ awaitFileClose(igfs.asSecondary(), FILE);
if (err.get() != null)
throw err.get();
@@ -1213,7 +1213,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrowsInherited(log(), new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1242,7 +1242,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendRenameNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1264,9 +1264,9 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @throws Exception If failed.
*/
public void testAppendRenameParentNoClose() throws Exception {
- create(igfs, paths(DIR, SUBDIR), null);
+ create(igfs.asSecondary(), paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1290,7 +1290,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendDeleteNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1322,7 +1322,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
public void testAppendDeleteParentNoClose() throws Exception {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
GridTestUtils.assertThrows(log, new Callable<Object>() {
@Override public Object call() throws Exception {
@@ -1361,7 +1361,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
create(igfs, paths(DIR, SUBDIR), null);
- createFile(igfs, FILE, false);
+ createFile(igfs.asSecondary(), FILE, false);
IgfsOutputStream os = null;
@@ -1389,7 +1389,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
int threadCnt = 10;
for (int i = 0; i < threadCnt; i++)
- createFile(igfs, new IgfsPath("/file" + i), false);
+ createFile(igfs.asSecondary(), new IgfsPath("/file" + i), false);
multithreaded(new Runnable() {
@Override public void run() {
@@ -1412,7 +1412,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
assert igfs.exists(path);
}
- awaitFileClose(igfs, path);
+ awaitFileClose(igfs.asSecondary(), path);
checkFileContent(igfs, path, chunks);
}
@@ -1484,7 +1484,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
fut.get();
- awaitFileClose(igfs, FILE);
+ awaitFileClose(igfs.asSecondary(), FILE);
if (err.get() != null)
throw err.get();
@@ -2138,6 +2138,18 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param files Files.
* @throws Exception If failed.
*/
+ public static void create(IgfsImpl igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files) throws Exception {
+ create(igfs.asSecondary(), dirs, files);
+ }
+
+ /**
+ * Create the given directories and files in the given IGFS.
+ *
+ * @param igfs IGFS.
+ * @param dirs Directories.
+ * @param files Files.
+ * @throws Exception If failed.
+ */
public static void create(IgfsSecondaryFileSystem igfs, @Nullable IgfsPath[] dirs, @Nullable IgfsPath[] files)
throws Exception {
if (dirs != null) {
@@ -2189,7 +2201,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void createFile(IgniteFileSystem igfs, IgfsPath file, boolean overwrite, long blockSize,
+ protected void createFile(IgfsImpl igfs, IgfsPath file, boolean overwrite, long blockSize,
@Nullable byte[]... chunks) throws Exception {
IgfsOutputStream os = null;
@@ -2201,7 +2213,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
finally {
U.closeQuiet(os);
- awaitFileClose(igfs, file);
+ awaitFileClose(igfs.asSecondary(), file);
}
}
@@ -2213,7 +2225,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
* @param chunks Data chunks.
* @throws Exception If failed.
*/
- protected void appendFile(IgniteFileSystem igfs, IgfsPath file, @Nullable byte[]... chunks)
+ protected void appendFile(IgfsImpl igfs, IgfsPath file, @Nullable byte[]... chunks)
throws Exception {
IgfsOutputStream os = null;
@@ -2225,7 +2237,7 @@ public abstract class IgfsAbstractSelfTest extends IgfsCommonAbstractTest {
finally {
U.closeQuiet(os);
- awaitFileClose(igfs, file);
+ awaitFileClose(igfs.asSecondary(), file);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
new file mode 100644
index 0000000..e4b2913
--- /dev/null
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsCachePerBlockLruEvictionPolicySelfTest.java
@@ -0,0 +1,485 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.processors.igfs;
+
+import org.apache.ignite.*;
+import org.apache.ignite.cache.*;
+import org.apache.ignite.cache.eviction.igfs.*;
+import org.apache.ignite.configuration.*;
+import org.apache.ignite.igfs.*;
+import org.apache.ignite.internal.*;
+import org.apache.ignite.internal.processors.cache.*;
+import org.apache.ignite.internal.util.lang.*;
+import org.apache.ignite.internal.util.typedef.*;
+import org.apache.ignite.internal.util.typedef.internal.*;
+import org.apache.ignite.spi.discovery.tcp.*;
+import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.*;
+import org.apache.ignite.testframework.*;
+
+import java.util.*;
+import java.util.concurrent.*;
+
+import static org.apache.ignite.cache.CacheAtomicityMode.*;
+import static org.apache.ignite.cache.CacheMode.*;
+import static org.apache.ignite.igfs.IgfsMode.*;
+
+/**
+ * Tests for IGFS per-block LR eviction policy.
+ */
+@SuppressWarnings({"ConstantConditions", "ThrowableResultOfMethodCallIgnored"})
+public class IgfsCachePerBlockLruEvictionPolicySelfTest extends IgfsCommonAbstractTest {
+ /** Primary IGFS name. */
+ private static final String IGFS_PRIMARY = "igfs-primary";
+
+ /** Primary IGFS name. */
+ private static final String IGFS_SECONDARY = "igfs-secondary";
+
+ /** Secondary file system REST endpoint configuration map. */
+ private static final Map<String, String> SECONDARY_REST_CFG = new HashMap<String, String>() {{
+ put("type", "tcp");
+ put("port", "11500");
+ }};
+
+ /** File working in PRIMARY mode. */
+ public static final IgfsPath FILE = new IgfsPath("/file");
+
+ /** File working in DUAL mode. */
+ public static final IgfsPath FILE_RMT = new IgfsPath("/fileRemote");
+
+ /** Primary IGFS instances. */
+ private static IgfsImpl igfsPrimary;
+
+ /** Secondary IGFS instance. */
+ private static IgfsImpl secondaryFs;
+
+ /** Primary file system data cache. */
+ private static GridCacheAdapter<IgfsBlockKey, byte[]> dataCache;
+
+ /** Eviction policy */
+ private static CacheIgfsPerBlockLruEvictionPolicy evictPlc;
+
+ /**
+ * Start a grid with the primary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startPrimary() throws Exception {
+ IgfsConfiguration igfsCfg = new IgfsConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(IGFS_PRIMARY);
+ igfsCfg.setBlockSize(512);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setPrefetchBlocks(1);
+ igfsCfg.setSequentialReadsBeforePrefetch(Integer.MAX_VALUE);
+ igfsCfg.setSecondaryFileSystem(secondaryFs.asSecondary());
+
+ Map<String, IgfsMode> pathModes = new HashMap<>();
+
+ pathModes.put(FILE_RMT.toString(), DUAL_SYNC);
+
+ igfsCfg.setPathModes(pathModes);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ evictPlc = new CacheIgfsPerBlockLruEvictionPolicy();
+
+ dataCacheCfg.setEvictionPolicy(evictPlc);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid-primary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setIgfsConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ igfsPrimary = (IgfsImpl)g.fileSystem(IGFS_PRIMARY);
+
+ dataCache = igfsPrimary.context().kernalContext().cache().internalCache(
+ igfsPrimary.context().configuration().getDataCacheName());
+ }
+
+ /**
+ * Start a grid with the secondary file system.
+ *
+ * @throws Exception If failed.
+ */
+ private void startSecondary() throws Exception {
+ IgfsConfiguration igfsCfg = new IgfsConfiguration();
+
+ igfsCfg.setDataCacheName("dataCache");
+ igfsCfg.setMetaCacheName("metaCache");
+ igfsCfg.setName(IGFS_SECONDARY);
+ igfsCfg.setBlockSize(512);
+ igfsCfg.setDefaultMode(PRIMARY);
+ igfsCfg.setIpcEndpointConfiguration(SECONDARY_REST_CFG);
+
+ CacheConfiguration dataCacheCfg = defaultCacheConfiguration();
+
+ dataCacheCfg.setName("dataCache");
+ dataCacheCfg.setCacheMode(PARTITIONED);
+ dataCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ dataCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ dataCacheCfg.setAffinityMapper(new IgfsGroupDataBlocksKeyMapper(128));
+ dataCacheCfg.setBackups(0);
+ dataCacheCfg.setQueryIndexEnabled(false);
+ dataCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ CacheConfiguration metaCacheCfg = defaultCacheConfiguration();
+
+ metaCacheCfg.setName("metaCache");
+ metaCacheCfg.setCacheMode(REPLICATED);
+ metaCacheCfg.setDistributionMode(CacheDistributionMode.PARTITIONED_ONLY);
+ metaCacheCfg.setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC);
+ metaCacheCfg.setQueryIndexEnabled(false);
+ metaCacheCfg.setAtomicityMode(TRANSACTIONAL);
+
+ IgniteConfiguration cfg = new IgniteConfiguration();
+
+ cfg.setGridName("grid-secondary");
+
+ TcpDiscoverySpi discoSpi = new TcpDiscoverySpi();
+
+ discoSpi.setIpFinder(new TcpDiscoveryVmIpFinder(true));
+
+ cfg.setDiscoverySpi(discoSpi);
+ cfg.setCacheConfiguration(dataCacheCfg, metaCacheCfg);
+ cfg.setIgfsConfiguration(igfsCfg);
+
+ cfg.setLocalHost("127.0.0.1");
+ cfg.setConnectorConfiguration(null);
+
+ Ignite g = G.start(cfg);
+
+ secondaryFs = (IgfsImpl)g.fileSystem(IGFS_SECONDARY);
+ }
+
+ /** {@inheritDoc} */
+ @Override protected void afterTest() throws Exception {
+ try {
+ // Cleanup.
+ igfsPrimary.format();
+
+ while (!dataCache.isEmpty())
+ U.sleep(100);
+
+ checkEvictionPolicy(0, 0);
+ }
+ finally {
+ stopAllGrids(false);
+ }
+ }
+
+ /**
+ * Startup primary and secondary file systems.
+ *
+ * @throws Exception If failed.
+ */
+ private void start() throws Exception {
+ startSecondary();
+ startPrimary();
+
+ evictPlc.setMaxBlocks(0);
+ evictPlc.setMaxSize(0);
+ evictPlc.setExcludePaths(null);
+ }
+
+ /**
+ * Test how evictions are handled for a file working in PRIMARY mode.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFilePrimary() throws Exception {
+ start();
+
+ // Create file in primary mode. It must not be propagated to eviction policy.
+ igfsPrimary.create(FILE, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE).blockSize();
+
+ append(FILE, blockSize);
+
+ checkEvictionPolicy(0, 0);
+
+ read(FILE, 0, blockSize);
+
+ checkEvictionPolicy(0, 0);
+ }
+
+ /**
+ * Test how evictions are handled for a file working in PRIMARY mode.
+ *
+ * @throws Exception If failed.
+ */
+ public void testFileDual() throws Exception {
+ start();
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ // File write.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(1, blockSize);
+
+ // One more write.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(2, blockSize * 2);
+
+ // Read.
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(2, blockSize * 2);
+ }
+
+ /**
+ * Ensure that a DUAL mode file is not propagated to eviction policy
+ *
+ * @throws Exception If failed.
+ */
+ public void testFileDualExclusion() throws Exception {
+ start();
+
+ evictPlc.setExcludePaths(Collections.singleton(FILE_RMT.toString()));
+
+ // Create file in primary mode. It must not be propagated to eviction policy.
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(0, 0);
+
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(0, 0);
+ }
+
+ /**
+ * Ensure that exception is thrown in case we are trying to rename file with one exclude setting to the file with
+ * another.
+ *
+ * @throws Exception If failed.
+ */
+ public void testRenameDifferentExcludeSettings() throws Exception {
+ start();
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ igfsPrimary.rename(FILE, FILE_RMT);
+
+ return null;
+ }
+ }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
+ "(need to copy and remove)");
+
+ GridTestUtils.assertThrows(log, new Callable<Object>() {
+ @Override public Object call() throws Exception {
+ igfsPrimary.rename(FILE_RMT, FILE);
+
+ return null;
+ }
+ }, IgfsInvalidPathException.class, "Cannot move file to a path with different eviction exclude setting " +
+ "(need to copy and remove)");
+ }
+
+ /**
+ * Test eviction caused by too much blocks.
+ *
+ * @throws Exception If failed.
+ */
+ public void testBlockCountEviction() throws Exception {
+ start();
+
+ int blockCnt = 3;
+
+ evictPlc.setMaxBlocks(blockCnt);
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ checkEvictionPolicy(0, 0);
+
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ // Write blocks up to the limit.
+ append(FILE_RMT, blockSize * blockCnt);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Write one more block what should cause eviction.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the first block.
+ read(FILE_RMT, 0, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+ checkMetrics(1, 1);
+ }
+
+ /**
+ * Test eviction caused by too big data size.
+ *
+ * @throws Exception If failed.
+ */
+ public void testDataSizeEviction() throws Exception {
+ start();
+
+ igfsPrimary.create(FILE_RMT, true).close();
+
+ int blockCnt = 3;
+ int blockSize = igfsPrimary.info(FILE_RMT).blockSize();
+
+ evictPlc.setMaxSize(blockSize * blockCnt);
+
+ // Write blocks up to the limit.
+ append(FILE_RMT, blockSize * blockCnt);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Reset metrics.
+ igfsPrimary.resetMetrics();
+
+ // Read the first block what should cause reordering.
+ read(FILE_RMT, 0, blockSize);
+
+ checkMetrics(1, 0);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Write one more block what should cause eviction of the block 2.
+ append(FILE_RMT, blockSize);
+
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the first block.
+ read(FILE_RMT, 0, blockSize);
+
+ checkMetrics(2, 0);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+
+ // Read the second block (which was evicted).
+ read(FILE_RMT, blockSize, blockSize);
+
+ checkMetrics(3, 1);
+ checkEvictionPolicy(blockCnt, blockCnt * blockSize);
+ }
+
+ /**
+ * Read some data from the given file with the given offset.
+ *
+ * @param path File path.
+ * @param off Offset.
+ * @param len Length.
+ * @throws Exception If failed.
+ */
+ private void read(IgfsPath path, int off, int len) throws Exception {
+ IgfsInputStream is = igfsPrimary.open(path);
+
+ is.readFully(off, new byte[len]);
+
+ is.close();
+ }
+
+ /**
+ * Append some data to the given file.
+ *
+ * @param path File path.
+ * @param len Data length.
+ * @throws Exception If failed.
+ */
+ private void append(IgfsPath path, int len) throws Exception {
+ IgfsOutputStream os = igfsPrimary.append(path, false);
+
+ os.write(new byte[len]);
+
+ os.close();
+ }
+
+ /**
+ * Check metrics counters.
+ *
+ * @param blocksRead Expected blocks read.
+ * @param blocksReadRmt Expected blocks read remote.
+ * @throws Exception If failed.
+ */
+ public void checkMetrics(final long blocksRead, final long blocksReadRmt) throws Exception {
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ IgfsMetrics metrics = igfsPrimary.metrics();
+
+ return metrics.blocksReadTotal() == blocksRead && metrics.blocksReadRemote() == blocksReadRmt;
+ }
+ }, 5000) : "Unexpected metrics [expectedBlocksReadTotal=" + blocksRead + ", actualBlocksReadTotal=" +
+ igfsPrimary.metrics().blocksReadTotal() + ", expectedBlocksReadRemote=" + blocksReadRmt +
+ ", actualBlocksReadRemote=" + igfsPrimary.metrics().blocksReadRemote() + ']';
+ }
+
+ /**
+ * Check eviction policy state.
+ *
+ * @param curBlocks Current blocks.
+ * @param curBytes Current bytes.
+ */
+ private void checkEvictionPolicy(final int curBlocks, final long curBytes) throws IgniteInterruptedCheckedException {
+ assert GridTestUtils.waitForCondition(new GridAbsPredicate() {
+ @Override public boolean apply() {
+ return evictPlc.getCurrentBlocks() == curBlocks && evictPlc.getCurrentSize() == curBytes;
+ }
+ }, 5000) : "Unexpected counts [expectedBlocks=" + curBlocks + ", actualBlocks=" + evictPlc.getCurrentBlocks() +
+ ", expectedBytes=" + curBytes + ", currentBytes=" + curBytes + ']';
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
index 2e48648..2af01bf 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDataManagerSelfTest.java
@@ -24,7 +24,6 @@ import org.apache.ignite.igfs.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.processors.cache.*;
import org.apache.ignite.internal.processors.cache.transactions.*;
-import org.apache.ignite.internal.util.*;
import org.apache.ignite.internal.util.typedef.*;
import org.apache.ignite.internal.util.typedef.internal.*;
import org.apache.ignite.lang.*;
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 3a0f90a..f44a988 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -1120,7 +1120,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
create(igfsSecondary, paths(DIR, SUBDIR), null);
create(igfs, null, null);
- createFile(igfsSecondary, FILE, true, chunk);
+ createFile(igfsSecondary.asSecondary(), FILE, true, chunk);
checkFileContent(igfs, FILE, chunk);
}
@@ -1148,7 +1148,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
out.close();
- awaitFileClose(igfsSecondary, FILE);
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
// Read the first block.
int totalRead = 0;
@@ -1227,7 +1227,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
out.close();
- awaitFileClose(igfsSecondary, FILE);
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
// Read the first two blocks.
int totalRead = 0;
@@ -1292,7 +1292,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
igfsSecondary.update(SUBDIR, props);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
// Ensure that directory structure was created.
checkExist(igfs, igfsSecondary, SUBDIR);
@@ -1317,7 +1317,7 @@ public abstract class IgfsDualAbstractSelfTest extends IgfsAbstractSelfTest {
igfsSecondary.update(DIR, propsDir);
igfsSecondary.update(SUBDIR, propsSubDir);
- createFile(igfs, FILE, true, chunk);
+ createFile(igfs.asSecondary(), FILE, true, chunk);
checkExist(igfs, igfsSecondary, SUBDIR);
checkFile(igfs, igfsSecondary, FILE, chunk);
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
index d6caf27..0739148 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsMetricsSelfTest.java
@@ -59,7 +59,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
private static IgniteFileSystem[] igfsPrimary;
/** Secondary IGFS instance. */
- private static IgniteFileSystem igfsSecondary;
+ private static IgfsImpl igfsSecondary;
/** Primary file system block size. */
public static final int PRIMARY_BLOCK_SIZE = 512;
@@ -108,7 +108,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setName(IGFS_PRIMARY);
igfsCfg.setBlockSize(PRIMARY_BLOCK_SIZE);
igfsCfg.setDefaultMode(PRIMARY);
- igfsCfg.setSecondaryFileSystem(igfsSecondary);
+ igfsCfg.setSecondaryFileSystem(igfsSecondary.asSecondary());
Map<String, IgfsMode> pathModes = new HashMap<>();
@@ -204,7 +204,7 @@ public class IgfsMetricsSelfTest extends IgfsCommonAbstractTest {
Ignite g = G.start(cfg);
- igfsSecondary = g.fileSystem(IGFS_SECONDARY);
+ igfsSecondary = (IgfsImpl)g.fileSystem(IGFS_SECONDARY);
}
/** @throws Exception If failed. */
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
index 2b501f7..1a37e93 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsModesSelfTest.java
@@ -97,7 +97,7 @@ public class IgfsModesSelfTest extends IgfsCommonAbstractTest {
igfsCfg.setPathModes(pathModes);
if (setSecondaryFs)
- igfsCfg.setSecondaryFileSystem(igfsSecondary);
+ igfsCfg.setSecondaryFileSystem(igfsSecondary.asSecondary());
CacheConfiguration cacheCfg = defaultCacheConfiguration();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
index 8f06117..cea510a 100644
--- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
+++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteIgfsTestSuite.java
@@ -48,7 +48,7 @@ public class IgniteIgfsTestSuite extends TestSuite {
if (U.isWindows())
suite.addTest(new TestSuite(IgfsServerManagerIpcEndpointRegistrationOnWindowsSelfTest.class));
- suite.addTest(new TestSuite(GridCacheIgfsPerBlockLruEvictionPolicySelfTest.class));
+ suite.addTest(new TestSuite(IgfsCachePerBlockLruEvictionPolicySelfTest.class));
suite.addTest(new TestSuite(IgfsStreamsSelfTest.class));
suite.addTest(new TestSuite(IgfsModesSelfTest.class));
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
index a86b335..a7fa88e 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/igfs/HadoopIgfsDualAbstractSelfTest.java
@@ -236,7 +236,7 @@ public abstract class HadoopIgfsDualAbstractSelfTest extends IgfsCommonAbstractT
out.close();
- awaitFileClose(igfsSecondary, FILE);
+ awaitFileClose(igfsSecondary.asSecondary(), FILE);
// Instantiate file system with overridden "seq reads before prefetch" property.
Configuration cfg = new Configuration();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/d83c88af/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
index f43419c..318dfb9 100644
--- a/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
+++ b/modules/hadoop/src/test/java/org/apache/ignite/internal/processors/hadoop/HadoopDefaultMapReducePlannerSelfTest.java
@@ -23,6 +23,7 @@ import org.apache.ignite.cluster.*;
import org.apache.ignite.configuration.*;
import org.apache.ignite.igfs.*;
import org.apache.ignite.igfs.mapreduce.*;
+import org.apache.ignite.igfs.secondary.*;
import org.apache.ignite.internal.*;
import org.apache.ignite.internal.cluster.*;
import org.apache.ignite.internal.processors.cache.*;
@@ -815,11 +816,6 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes
}
/** {@inheritDoc} */
- @Override public Map<String, String> properties() {
- return Collections.emptyMap();
- }
-
- /** {@inheritDoc} */
@Override public IgfsOutputStream create(IgfsPath path, boolean overwrite) {
return null;
}
@@ -916,6 +912,11 @@ public class HadoopDefaultMapReducePlannerSelfTest extends HadoopAbstractSelfTes
@Override public <R> IgniteFuture<R> future() {
return null;
}
+
+ /** {@inheritDoc} */
+ @Override public IgfsSecondaryFileSystem asSecondary() {
+ return null;
+ }
}
/**
[4/6] incubator-ignite git commit: # IGNITE-386: WIP.
Posted by vo...@apache.org.
# IGNITE-386: WIP.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/7ef91986
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/7ef91986
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/7ef91986
Branch: refs/heads/ignite-386
Commit: 7ef919863e642266f4aa1095393c9414f6e3e237
Parents: dd06414
Author: vozerov-gridgain <vo...@gridgain.com>
Authored: Tue Mar 3 17:58:21 2015 +0300
Committer: vozerov-gridgain <vo...@gridgain.com>
Committed: Tue Mar 3 17:58:21 2015 +0300
----------------------------------------------------------------------
.../main/java/org/apache/ignite/IgniteFileSystem.java | 3 ---
.../ignite/igfs/secondary/IgfsSecondaryFileSystem.java | 9 ---------
.../apache/ignite/internal/processors/igfs/IgfsEx.java | 12 ++++++++++++
.../internal/processors/igfs/IgfsOutputStreamImpl.java | 2 +-
.../internal/processors/igfs/IgfsAbstractSelfTest.java | 2 +-
.../processors/igfs/IgfsDualAbstractSelfTest.java | 2 +-
.../internal/processors/igfs/IgfsStreamsSelfTest.java | 2 +-
.../hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java | 2 ++
.../ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java | 2 +-
.../ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java | 2 +-
.../processors/hadoop/igfs/HadoopIgfsProperties.java | 2 +-
11 files changed, 21 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
index 893bca8..280628d 100644
--- a/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/IgniteFileSystem.java
@@ -61,9 +61,6 @@ public interface IgniteFileSystem extends IgfsSecondaryFileSystem, IgniteAsyncSu
/** IGFS scheme name. */
public static final String IGFS_SCHEME = "igfs";
- /** File property: prefer writes to local node. */
- public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
-
/**
* Gets IGFS name.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
index 88fcae4..f306d67 100644
--- a/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
+++ b/modules/core/src/main/java/org/apache/ignite/igfs/secondary/IgfsSecondaryFileSystem.java
@@ -35,15 +35,6 @@ import java.util.*;
* This is the minimum of functionality that is needed to work as secondary file system in dual modes of IGFS.
*/
public interface IgfsSecondaryFileSystem {
- /** File property: user name. */
- public static final String PROP_USER_NAME = "usrName";
-
- /** File property: group name. */
- public static final String PROP_GROUP_NAME = "grpName";
-
- /** File property: permission. */
- public static final String PROP_PERMISSION = "permission";
-
/**
* Checks if the specified path exists in the file system.
*
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
index f5579b9..7e96c19 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsEx.java
@@ -29,6 +29,18 @@ import java.net.*;
* Internal API extension for {@link org.apache.ignite.IgniteFileSystem}.
*/
public interface IgfsEx extends IgniteFileSystem {
+ /** File property: user name. */
+ public static final String PROP_USER_NAME = "usrName";
+
+ /** File property: group name. */
+ public static final String PROP_GROUP_NAME = "grpName";
+
+ /** File property: permission. */
+ public static final String PROP_PERMISSION = "permission";
+
+ /** File property: prefer writes to local node. */
+ public static final String PROP_PREFER_LOCAL_WRITES = "locWrite";
+
/**
* Stops IGFS cleaning all used resources.
*/
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
----------------------------------------------------------------------
diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
index 49e2f68..6c6dd9d 100644
--- a/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
+++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/igfs/IgfsOutputStreamImpl.java
@@ -405,7 +405,7 @@ class IgfsOutputStreamImpl extends IgfsOutputStreamAdapter {
if (!igfsCtx.configuration().isFragmentizerEnabled())
return null;
- if (!Boolean.parseBoolean(fileInfo.properties().get(IgniteFileSystem.PROP_PREFER_LOCAL_WRITES)))
+ if (!Boolean.parseBoolean(fileInfo.properties().get(IgfsEx.PROP_PREFER_LOCAL_WRITES)))
return null;
int blockSize = fileInfo.blockSize();
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
index 7bc7694..12ef0c1 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsAbstractSelfTest.java
@@ -38,11 +38,11 @@ import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.cache.CacheAtomicityMode.*;
import static org.apache.ignite.cache.CacheMemoryMode.*;
import static org.apache.ignite.cache.CacheMode.*;
import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* Test fo regular igfs operations.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
index 1566cca..3a0f90a 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsDualAbstractSelfTest.java
@@ -28,8 +28,8 @@ import java.io.*;
import java.util.*;
import java.util.concurrent.*;
-import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.igfs.IgfsMode.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* Tests for IGFS working in mode when remote file system exists: DUAL_SYNC, DUAL_ASYNC.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
----------------------------------------------------------------------
diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
index fa8bf89..38d4af5 100644
--- a/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
+++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/igfs/IgfsStreamsSelfTest.java
@@ -234,7 +234,7 @@ public class IgfsStreamsSelfTest extends IgfsCommonAbstractTest {
IgniteFileSystem fs2 = grid(2).fileSystem("igfs");
try (IgfsOutputStream out = fs0.create(path, 128, false, 1, CFG_GRP_SIZE,
- F.asMap(IgniteFileSystem.PROP_PREFER_LOCAL_WRITES, "true"))) {
+ F.asMap(IgfsEx.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
index 65d1901..3e5f32f 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/IgniteHadoopIgfsSecondaryFileSystem.java
@@ -35,6 +35,8 @@ import java.io.*;
import java.net.*;
import java.util.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
+
/**
* Adapter to use any Hadoop file system {@link org.apache.hadoop.fs.FileSystem} as
* {@link org.apache.ignite.igfs.secondary.IgfsSecondaryFileSystem}.
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
index 9261273..2ee9b50 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v1/IgniteHadoopFileSystem.java
@@ -39,11 +39,11 @@ import java.net.*;
import java.util.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.configuration.IgfsConfiguration.*;
import static org.apache.ignite.igfs.IgfsMode.*;
import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* {@code IGFS} Hadoop 1.x file system driver over file system API. To use
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
index fa1abb5..bcf3029 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/hadoop/fs/v2/IgniteHadoopFileSystem.java
@@ -39,11 +39,11 @@ import java.net.*;
import java.util.*;
import java.util.concurrent.atomic.*;
-import static org.apache.ignite.IgniteFileSystem.*;
import static org.apache.ignite.configuration.IgfsConfiguration.*;
import static org.apache.ignite.igfs.IgfsMode.*;
import static org.apache.ignite.internal.processors.hadoop.fs.HadoopParameters.*;
import static org.apache.ignite.internal.processors.hadoop.igfs.HadoopIgfsUtils.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* {@code IGFS} Hadoop 2.x file system driver over file system API. To use
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/7ef91986/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
----------------------------------------------------------------------
diff --git a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
index 2cbf32f..54e87dd 100644
--- a/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
+++ b/modules/hadoop/src/main/java/org/apache/ignite/internal/processors/hadoop/igfs/HadoopIgfsProperties.java
@@ -22,7 +22,7 @@ import org.apache.ignite.*;
import java.util.*;
-import static org.apache.ignite.IgniteFileSystem.*;
+import static org.apache.ignite.internal.processors.igfs.IgfsEx.*;
/**
* Hadoop file system properties.