You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by up...@apache.org on 2015/08/20 23:47:11 UTC
incubator-geode git commit: Adding a couple of simple junit tests for
FileSystem
Repository: incubator-geode
Updated Branches:
refs/heads/feature/GEODE-11 a376120dd -> ebb357ec1
Adding a couple of simple junit tests for FileSystem
Testing some basic file write and read operations, as well as file
renames and deletes.
I also added some more javadocs to the filesystem classes and moved them
into a separate package.
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/ebb357ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/ebb357ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/ebb357ec
Branch: refs/heads/feature/GEODE-11
Commit: ebb357ec159a930608cef63be79d4e56c2e9e0f7
Parents: a376120
Author: Dan Smith <up...@apache.org>
Authored: Thu Aug 20 12:56:02 2015 -0700
Committer: Dan Smith <up...@apache.org>
Committed: Thu Aug 20 14:46:42 2015 -0700
----------------------------------------------------------------------
gemfire-lucene/build.gradle | 2 +-
.../gemfire/cache/lucene/internal/ChunkKey.java | 66 -------
.../gemfire/cache/lucene/internal/File.java | 74 --------
.../cache/lucene/internal/FileInputStream.java | 97 ----------
.../cache/lucene/internal/FileOutputStream.java | 70 -------
.../cache/lucene/internal/FileSystem.java | 111 -----------
.../cache/lucene/internal/RegionDirectory.java | 3 +
.../lucene/internal/filesystem/ChunkKey.java | 69 +++++++
.../cache/lucene/internal/filesystem/File.java | 86 +++++++++
.../internal/filesystem/FileInputStream.java | 103 +++++++++++
.../internal/filesystem/FileOutputStream.java | 74 ++++++++
.../lucene/internal/filesystem/FileSystem.java | 130 +++++++++++++
.../filesystem/FileSystemJUnitTest.java | 182 +++++++++++++++++++
13 files changed, 648 insertions(+), 419 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/build.gradle
----------------------------------------------------------------------
diff --git a/gemfire-lucene/build.gradle b/gemfire-lucene/build.gradle
index b360c59..fdc1e4e 100644
--- a/gemfire-lucene/build.gradle
+++ b/gemfire-lucene/build.gradle
@@ -5,5 +5,5 @@ dependencies {
compile 'org.apache.lucene:lucene-queries:5.0.0'
compile 'org.apache.lucene:lucene-queryparser:5.0.0'
- testCompile project(path: ':gemfire-junit', configuration: 'testOutput')
+ provided project(path: ':gemfire-junit', configuration: 'testOutput')
}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/ChunkKey.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/ChunkKey.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/ChunkKey.java
deleted file mode 100644
index 86d9bcb..0000000
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/ChunkKey.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package com.gemstone.gemfire.cache.lucene.internal;
-
-import java.io.Serializable;
-
-public class ChunkKey implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- String fileName;
- int chunkId;
-
- ChunkKey(String fileName, int chunkId) {
- this.fileName = fileName;
- this.chunkId = chunkId;
- }
-
- /**
- * @return the fileName
- */
- public String getFileName() {
- return fileName;
- }
-
- /**
- * @return the chunkId
- */
- public int getChunkId() {
- return chunkId;
- }
-
- @Override
- public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + fileName.hashCode();
- result = prime * result + chunkId;
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof ChunkKey)) {
- return false;
- }
- ChunkKey other = (ChunkKey) obj;
- if (chunkId != other.chunkId) {
- return false;
- }
- if (fileName == null) {
- if (other.fileName != null) {
- return false;
- }
- } else if (!fileName.equals(other.fileName)) {
- return false;
- }
- return true;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/File.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/File.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/File.java
deleted file mode 100644
index 9df6641..0000000
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/File.java
+++ /dev/null
@@ -1,74 +0,0 @@
-package com.gemstone.gemfire.cache.lucene.internal;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
-
-final public class File implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private transient FileSystem fileSystem;
- private transient int chunkSize;
-
- String name;
- long length = 0;
- int chunks = 0;
- long created = System.currentTimeMillis();
- long modified = created;
-
- File(final FileSystem fileSystem, final String name) {
- setFileSystem(fileSystem);
-
- this.name = name;
- }
-
- /**
- * @return the name
- */
- public String getName() {
- return name;
- }
-
- /**
- * @return the length
- */
- public long getLength() {
- return length;
- }
-
- /**
- * @return the created
- */
- public long getCreated() {
- return created;
- }
-
- /**
- * @return the modified
- */
- public long getModified() {
- return modified;
- }
-
- public InputStream getInputStream() {
- // TODO get read lock?
- return new FileInputStream(this);
- }
-
- public OutputStream getOutputStream() {
- return new FileOutputStream(this);
- }
-
- void setFileSystem(final FileSystem fileSystem) {
- this.fileSystem = fileSystem;
- this.chunkSize = fileSystem.chunkSize;
- }
-
- int getChunkSize() {
- return chunkSize;
- }
-
- public FileSystem getFileSystem() {
- return fileSystem;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileInputStream.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileInputStream.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileInputStream.java
deleted file mode 100644
index 0565974..0000000
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileInputStream.java
+++ /dev/null
@@ -1,97 +0,0 @@
-package com.gemstone.gemfire.cache.lucene.internal;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-final class FileInputStream extends InputStream {
-
- private final File file;
- private byte[] chunk = null;
- private int chunkPosition = 0;
- private int chunkId = 0;
- private boolean open = true;
-
- public FileInputStream(File file) {
- this.file = file;
- nextChunk();
- }
-
- @Override
- public int read() throws IOException {
- assertOpen();
-
- checkAndFetchNextChunk();
-
- if (null == chunk) {
- return -1;
- }
-
- return chunk[chunkPosition++] & 0xff;
- }
-
- @Override
- public int read(byte[] b, int off, int len) throws IOException {
- assertOpen();
-
- checkAndFetchNextChunk();
-
- if (null == chunk) {
- return -1;
- }
-
- int read = 0;
- while (len > 0) {
- final int min = Math.min(remaining(), len);
- System.arraycopy(chunk, chunkPosition, b, off, min);
- off += min;
- len -= min;
- chunkPosition += min;
- read += min;
-
- if (len > 0) {
- // we read to the end of the chunk, fetch another.
- nextChunk();
- if (null == chunk) {
- break;
- }
- }
- }
-
- return read;
- }
-
- @Override
- public int available() throws IOException {
- assertOpen();
-
- return remaining();
- }
-
- @Override
- public void close() throws IOException {
- if (open) {
- open = false;
- }
- }
-
- private int remaining() {
- return chunk.length - chunkPosition;
- }
-
- private void checkAndFetchNextChunk() {
- if (null != chunk && remaining() <= 0) {
- nextChunk();
- }
- }
-
- private void nextChunk() {
- chunk = file.getFileSystem().getChunk(this.file, chunkId++);
- chunkPosition = 0;
- }
-
- private void assertOpen() throws IOException {
- if (!open) {
- throw new IOException("Closed");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileOutputStream.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileOutputStream.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileOutputStream.java
deleted file mode 100644
index 58d9572..0000000
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileOutputStream.java
+++ /dev/null
@@ -1,70 +0,0 @@
-package com.gemstone.gemfire.cache.lucene.internal;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-
-final class FileOutputStream extends OutputStream {
-
- private final File file;
- private ByteBuffer buffer;
- private boolean open = true;
-
- public FileOutputStream(final File file) {
- this.file = file;
- buffer = ByteBuffer.allocate(file.getChunkSize());
- }
-
- @Override
- public void write(final int b) throws IOException {
- assertOpen();
-
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- buffer.put((byte) b);
- file.length++;
- }
-
- @Override
- public void write(final byte[] b, int off, int len) throws IOException {
- assertOpen();
-
- while (len > 0) {
- if (buffer.remaining() == 0) {
- flushBuffer();
- }
-
- final int min = Math.min(buffer.remaining(), len);
- buffer.put(b, off, min);
- off += min;
- len -= min;
- file.length += min;
- }
- }
-
- @Override
- public void close() throws IOException {
- if (open) {
- flushBuffer();
- file.modified = System.currentTimeMillis();
- file.getFileSystem().updateFile(file);
- open = false;
- buffer = null;
- }
- }
-
- private void flushBuffer() {
- byte[] chunk = Arrays.copyOfRange(buffer.array(), buffer.arrayOffset(), buffer.position());
- file.getFileSystem().putChunk(file, file.chunks++, chunk);
- buffer.rewind();
- }
-
- private void assertOpen() throws IOException {
- if (!open) {
- throw new IOException("Closed");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileSystem.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileSystem.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileSystem.java
deleted file mode 100644
index 8bba0df..0000000
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/FileSystem.java
+++ /dev/null
@@ -1,111 +0,0 @@
-package com.gemstone.gemfire.cache.lucene.internal;
-
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collection;
-
-
-import com.gemstone.gemfire.cache.Region;
-
-public class FileSystem {
- // private final Cache cache;
- private final Region<String, File> fileRegion;
- private final Region<ChunkKey, byte[]> chunkRegion;
-
- final int chunkSize = 1_000_000;
-
- public FileSystem(Region<String, File> fileRegion, Region<ChunkKey, byte[]> chunkRegion) {
- super();
- this.fileRegion = fileRegion;
- this.chunkRegion = chunkRegion;
- }
-
- public Collection<String> listFileNames() {
- return fileRegion.keySet();
- }
-
- public File createFile(final String name) throws IOException {
- // TODO lock region ?
- final File file = new File(this, name);
- if (null != fileRegion.putIfAbsent(name, file)) {
- throw new IOException("File exists.");
- }
- // TODO unlock region ?
- return file;
- }
-
- public File getFile(final String name) throws FileNotFoundException {
- final File file = fileRegion.get(name);
-
- if (null == file) {
- throw new FileNotFoundException(name);
- }
-
- file.setFileSystem(this);
- return file;
- }
-
- public void deleteFile(final String name) {
- // TODO locks?
-
- // TODO consider removeAll with all ChunkKeys listed.
- final ChunkKey key = new ChunkKey(name, 0);
- while (true) {
- // TODO consider mutable ChunkKey
- if (null == chunkRegion.remove(key)) {
- // no more chunks
- break;
- }
- key.chunkId++;
- }
-
- fileRegion.remove(name);
- }
-
- public void renameFile(String source, String dest) throws IOException {
- final File destFile = createFile(dest);
-
- final File sourceFile = fileRegion.remove(source);
- if (null == sourceFile) {
- throw new FileNotFoundException(source);
- }
-
- destFile.chunks = sourceFile.chunks;
- destFile.created = sourceFile.created;
- destFile.length = sourceFile.length;
- destFile.modified = sourceFile.modified;
-
- // TODO copy on write?
- final ChunkKey sourceKey = new ChunkKey(source, 0);
- while (true) {
- byte[] chunk = chunkRegion.remove(sourceKey);
- if (null == chunk) {
- // no more chunks
- break;
- }
- putChunk(destFile, sourceKey.chunkId, chunk);
- sourceKey.chunkId++;
- }
-
- updateFile(destFile);
- }
-
- byte[] getChunk(final File file, final int id) {
- final ChunkKey key = new ChunkKey(file.getName(), id);
- final byte[] chunk = chunkRegion.get(key);
- return chunk;
- }
-
- public void putChunk(final File file, final int id, final byte[] chunk) {
- final ChunkKey key = new ChunkKey(file.getName(), id);
- chunkRegion.put(key, chunk);
- }
-
- void updateFile(File file) {
- fileRegion.put(file.getName(), file);
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/RegionDirectory.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/RegionDirectory.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/RegionDirectory.java
index 13700a7..ade257a 100644
--- a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/RegionDirectory.java
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/RegionDirectory.java
@@ -22,6 +22,9 @@ import com.gemstone.gemfire.cache.PartitionAttributesFactory;
import com.gemstone.gemfire.cache.Region;
import com.gemstone.gemfire.cache.RegionAttributes;
import com.gemstone.gemfire.cache.RegionShortcut;
+import com.gemstone.gemfire.cache.lucene.internal.filesystem.ChunkKey;
+import com.gemstone.gemfire.cache.lucene.internal.filesystem.File;
+import com.gemstone.gemfire.cache.lucene.internal.filesystem.FileSystem;
import com.gemstone.gemfire.internal.i18n.LocalizedStrings;
import com.gemstone.gemfire.internal.logging.LogService;
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/ChunkKey.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/ChunkKey.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/ChunkKey.java
new file mode 100644
index 0000000..5564c02
--- /dev/null
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/ChunkKey.java
@@ -0,0 +1,69 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import java.io.Serializable;
+
+/**
+ * The key for a single chunk on a file stored within a region.
+ */
+public class ChunkKey implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ String fileName;
+ int chunkId;
+
+ ChunkKey(String fileName, int chunkId) {
+ this.fileName = fileName;
+ this.chunkId = chunkId;
+ }
+
+ /**
+ * @return the fileName
+ */
+ public String getFileName() {
+ return fileName;
+ }
+
+ /**
+ * @return the chunkId
+ */
+ public int getChunkId() {
+ return chunkId;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + fileName.hashCode();
+ result = prime * result + chunkId;
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ChunkKey)) {
+ return false;
+ }
+ ChunkKey other = (ChunkKey) obj;
+ if (chunkId != other.chunkId) {
+ return false;
+ }
+ if (fileName == null) {
+ if (other.fileName != null) {
+ return false;
+ }
+ } else if (!fileName.equals(other.fileName)) {
+ return false;
+ }
+ return true;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/File.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/File.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/File.java
new file mode 100644
index 0000000..1f5edb7
--- /dev/null
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/File.java
@@ -0,0 +1,86 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+
+/**
+ * A file that is stored in a gemfire region.
+ */
+public class File implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private transient FileSystem fileSystem;
+ private transient int chunkSize;
+
+ private String name;
+ long length = 0;
+ int chunks = 0;
+ long created = System.currentTimeMillis();
+ long modified = created;
+
+ File(final FileSystem fileSystem, final String name) {
+ setFileSystem(fileSystem);
+
+ this.name = name;
+ }
+
+ /**
+ * @return the name
+ */
+ public String getName() {
+ return name;
+ }
+
+ /**
+ * @return the length
+ */
+ public long getLength() {
+ return length;
+ }
+
+ /**
+ * @return the created
+ */
+ public long getCreated() {
+ return created;
+ }
+
+ /**
+ * @return the modified
+ */
+ public long getModified() {
+ return modified;
+ }
+
+ /**
+ * Get an input stream that reads from the beginning the file
+ *
+ * The input stream is not threadsafe
+ */
+ public InputStream getInputStream() {
+ // TODO get read lock?
+ return new FileInputStream(this);
+ }
+
+ /**
+ * Get an output stream that appends to the end
+ * of the file.
+ */
+ public OutputStream getOutputStream() {
+ return new FileOutputStream(this);
+ }
+
+ void setFileSystem(final FileSystem fileSystem) {
+ this.fileSystem = fileSystem;
+ this.chunkSize = fileSystem.chunkSize;
+ }
+
+ int getChunkSize() {
+ return chunkSize;
+ }
+
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileInputStream.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileInputStream.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileInputStream.java
new file mode 100644
index 0000000..5304a55
--- /dev/null
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileInputStream.java
@@ -0,0 +1,103 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * An input stream that reads chunks from
+ * a File saved in the region. This input stream
+ * will keep going back to the region to look for
+ * chunks until nothing is found.
+ */
+final class FileInputStream extends InputStream {
+
+ private final File file;
+ private byte[] chunk = null;
+ private int chunkPosition = 0;
+ private int chunkId = 0;
+ private boolean open = true;
+
+ public FileInputStream(File file) {
+ this.file = file;
+ nextChunk();
+ }
+
+ @Override
+ public int read() throws IOException {
+ assertOpen();
+
+ checkAndFetchNextChunk();
+
+ if (null == chunk) {
+ return -1;
+ }
+
+ return chunk[chunkPosition++] & 0xff;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ assertOpen();
+
+ checkAndFetchNextChunk();
+
+ if (null == chunk) {
+ return -1;
+ }
+
+ int read = 0;
+ while (len > 0) {
+ final int min = Math.min(remaining(), len);
+ System.arraycopy(chunk, chunkPosition, b, off, min);
+ off += min;
+ len -= min;
+ chunkPosition += min;
+ read += min;
+
+ if (len > 0) {
+ // we read to the end of the chunk, fetch another.
+ nextChunk();
+ if (null == chunk) {
+ break;
+ }
+ }
+ }
+
+ return read;
+ }
+
+ @Override
+ public int available() throws IOException {
+ assertOpen();
+
+ return remaining();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (open) {
+ open = false;
+ }
+ }
+
+ private int remaining() {
+ return chunk.length - chunkPosition;
+ }
+
+ private void checkAndFetchNextChunk() {
+ if (null != chunk && remaining() <= 0) {
+ nextChunk();
+ }
+ }
+
+ private void nextChunk() {
+ chunk = file.getFileSystem().getChunk(this.file, chunkId++);
+ chunkPosition = 0;
+ }
+
+ private void assertOpen() throws IOException {
+ if (!open) {
+ throw new IOException("Closed");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileOutputStream.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileOutputStream.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileOutputStream.java
new file mode 100644
index 0000000..4006be2
--- /dev/null
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileOutputStream.java
@@ -0,0 +1,74 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+final class FileOutputStream extends OutputStream {
+
+ private final File file;
+ private ByteBuffer buffer;
+ private boolean open = true;
+
+ public FileOutputStream(final File file) {
+ this.file = file;
+ buffer = ByteBuffer.allocate(file.getChunkSize());
+ }
+
+ @Override
+ public void write(final int b) throws IOException {
+ assertOpen();
+
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ buffer.put((byte) b);
+ file.length++;
+ }
+
+ @Override
+ public void write(final byte[] b, int off, int len) throws IOException {
+ assertOpen();
+
+ // TODO - What is the state of the system if
+ // things crash without close?
+ // Seems like a file metadata will be out of sync
+
+ while (len > 0) {
+ if (buffer.remaining() == 0) {
+ flushBuffer();
+ }
+
+ final int min = Math.min(buffer.remaining(), len);
+ buffer.put(b, off, min);
+ off += min;
+ len -= min;
+ file.length += min;
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (open) {
+ flushBuffer();
+ file.modified = System.currentTimeMillis();
+ file.getFileSystem().updateFile(file);
+ open = false;
+ buffer = null;
+ }
+ }
+
+ private void flushBuffer() {
+ byte[] chunk = Arrays.copyOfRange(buffer.array(), buffer.arrayOffset(), buffer.position());
+ file.getFileSystem().putChunk(file, file.chunks++, chunk);
+ buffer.rewind();
+ }
+
+ private void assertOpen() throws IOException {
+ if (!open) {
+ throw new IOException("Closed");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystem.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystem.java b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystem.java
new file mode 100644
index 0000000..62b3700
--- /dev/null
+++ b/gemfire-lucene/src/main/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystem.java
@@ -0,0 +1,130 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * A Filesystem like interface that stores file data in gemfire regions.
+ *
+ * This filesystem is safe for use with multiple threads if the threads are not
+ * modifying the same files. A single file is not safe to modify by multiple
+ * threads, even between different members of the distributed system.
+ *
+ * Changes to a file may not be visible to other members of the system until the
+ * FileOutputStream is closed.
+ */
+public class FileSystem {
+ // private final Cache cache;
+ private final ConcurrentMap<String, File> fileRegion;
+ private final ConcurrentMap<ChunkKey, byte[]> chunkRegion;
+
+ final int chunkSize = 1_000_000;
+
+ public FileSystem(ConcurrentMap<String, File> fileRegion, ConcurrentMap<ChunkKey, byte[]> chunkRegion) {
+ super();
+ this.fileRegion = fileRegion;
+ this.chunkRegion = chunkRegion;
+ }
+
+ public Collection<String> listFileNames() {
+ return fileRegion.keySet();
+ }
+
+ public File createFile(final String name) throws IOException {
+ // TODO lock region ?
+ final File file = new File(this, name);
+ if (null != fileRegion.putIfAbsent(name, file)) {
+ throw new IOException("File exists.");
+ }
+ // TODO unlock region ?
+ return file;
+ }
+
+ public File getFile(final String name) throws FileNotFoundException {
+ final File file = fileRegion.get(name);
+
+ if (null == file) {
+ throw new FileNotFoundException(name);
+ }
+
+ file.setFileSystem(this);
+ return file;
+ }
+
+ public void deleteFile(final String name) {
+ // TODO locks?
+
+ // TODO - What is the state of the system if
+ // things crash in the middle of removing this file?
+ // Seems like a file will be left with some
+ // dangling chunks at the end of the file
+
+ // TODO consider removeAll with all ChunkKeys listed.
+ final ChunkKey key = new ChunkKey(name, 0);
+ while (true) {
+ // TODO consider mutable ChunkKey
+ if (null == chunkRegion.remove(key)) {
+ // no more chunks
+ break;
+ }
+ key.chunkId++;
+ }
+
+ fileRegion.remove(name);
+ }
+
+ public void renameFile(String source, String dest) throws IOException {
+ final File destFile = createFile(dest);
+
+ // TODO - What is the state of the system if
+ // things crash in the middle of moving this file?
+ // Seems like a file will be left with some
+ // dangling chunks at the end of the file
+
+ final File sourceFile = fileRegion.get(source);
+ if (null == sourceFile) {
+ throw new FileNotFoundException(source);
+ }
+
+ destFile.chunks = sourceFile.chunks;
+ destFile.created = sourceFile.created;
+ destFile.length = sourceFile.length;
+ destFile.modified = sourceFile.modified;
+
+ // TODO copy on write?
+ final ChunkKey sourceKey = new ChunkKey(source, 0);
+ while (true) {
+ byte[] chunk = chunkRegion.remove(sourceKey);
+ if (null == chunk) {
+ // no more chunks
+ break;
+ }
+ putChunk(destFile, sourceKey.chunkId, chunk);
+ sourceKey.chunkId++;
+ }
+
+ updateFile(destFile);
+ fileRegion.remove(source);
+ }
+
+ byte[] getChunk(final File file, final int id) {
+ final ChunkKey key = new ChunkKey(file.getName(), id);
+ final byte[] chunk = chunkRegion.get(key);
+ return chunk;
+ }
+
+ public void putChunk(final File file, final int id, final byte[] chunk) {
+ final ChunkKey key = new ChunkKey(file.getName(), id);
+ chunkRegion.put(key, chunk);
+ }
+
+ void updateFile(File file) {
+ fileRegion.put(file.getName(), file);
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/ebb357ec/gemfire-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystemJUnitTest.java
----------------------------------------------------------------------
diff --git a/gemfire-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystemJUnitTest.java b/gemfire-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystemJUnitTest.java
new file mode 100644
index 0000000..b79bd1e
--- /dev/null
+++ b/gemfire-lucene/src/test/java/com/gemstone/gemfire/cache/lucene/internal/filesystem/FileSystemJUnitTest.java
@@ -0,0 +1,182 @@
+package com.gemstone.gemfire.cache.lucene.internal.filesystem;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.gemstone.gemfire.test.junit.categories.UnitTest;
+
+@Category(UnitTest.class)
+public class FileSystemJUnitTest {
+
+ private static final int SMALL_CHUNK = 523;
+ private static final int LARGE_CHUNK = 1024 * 1024 * 5;
+ private FileSystem system;
+ private Random rand = new Random();
+
+ @Before
+ public void setUp() {
+ ConcurrentHashMap<String, File> fileRegion = new ConcurrentHashMap<String, File>();
+ ConcurrentHashMap<ChunkKey, byte[]> chunkRegion = new ConcurrentHashMap<ChunkKey, byte[]>();
+ system = new FileSystem(fileRegion, chunkRegion);
+ }
+
+ @Test
+ public void testReadWriteBytes() throws IOException {
+ long start = System.currentTimeMillis();
+
+ File file1= system.createFile("testFile1");
+
+ assertEquals(0, file1.getLength());
+
+ OutputStream outputStream1 = file1.getOutputStream();
+
+ outputStream1.write(2);
+ byte[] data = new byte[LARGE_CHUNK];
+ rand.nextBytes(data);
+ outputStream1.write(data);
+ outputStream1.write(44);
+ outputStream1.close();
+
+ assertEquals(2 + LARGE_CHUNK, file1.getLength());
+ assertTrue(file1.getModified() >= start);
+
+ OutputStream outputStream2 = file1.getOutputStream();
+
+ outputStream2.write(123);
+ byte[] data2 = new byte[SMALL_CHUNK];
+ rand.nextBytes(data2);
+ outputStream2.write(data2);
+ outputStream2.close();
+
+ assertEquals(3 + LARGE_CHUNK + SMALL_CHUNK, file1.getLength());
+
+ InputStream is = file1.getInputStream();
+
+ assertEquals(2, is.read());
+ byte[] resultData = new byte[LARGE_CHUNK];
+ assertEquals(LARGE_CHUNK, is.read(resultData));
+ assertArrayEquals(data, resultData);
+ assertEquals(44, is.read());
+ assertEquals(123, is.read());
+
+
+ //Test read to an offset
+ Arrays.fill(resultData, (byte) 0);
+ assertEquals(SMALL_CHUNK, is.read(resultData, 50, SMALL_CHUNK));
+
+ //Make sure the data read matches
+ byte[] expectedData = new byte[LARGE_CHUNK];
+ Arrays.fill(expectedData, (byte) 0);
+ System.arraycopy(data2, 0, expectedData, 50, data2.length);
+ assertArrayEquals(expectedData, resultData);
+
+ assertEquals(-1, is.read());
+ assertEquals(-1, is.read(data));
+ is.close();
+
+ //Test the skip interface
+ is = file1.getInputStream();
+ is.skip(LARGE_CHUNK + 3);
+
+
+ Arrays.fill(resultData, (byte) 0);
+ assertEquals(SMALL_CHUNK, is.read(resultData));
+
+ Arrays.fill(expectedData, (byte) 0);
+ System.arraycopy(data2, 0, expectedData, 0, data2.length);
+ assertArrayEquals(expectedData, resultData);
+
+ assertEquals(-1, is.read());
+ }
+
+ @Test
+ public void testFileOperations() throws IOException {
+ String name1 = "testFile1";
+ File file1= system.createFile(name1);
+ byte[] file1Data = writeRandomBytes(file1);
+
+ String name2 = "testFile2";
+ File file2= system.createFile(name2);
+ byte[] file2Data = writeRandomBytes(file2);
+
+ file1 = system.getFile(name1);
+ file2 = system.getFile(name2);
+
+ assertEquals(new HashSet(Arrays.asList(name1, name2)), system.listFileNames());
+ assertContents(file1Data, file1);
+ assertContents(file2Data, file2);
+
+
+ try {
+ system.renameFile(name1, name2);
+ fail("Should have received an exception");
+ } catch(IOException expected) {
+
+ }
+ assertEquals(new HashSet(Arrays.asList(name1, name2)), system.listFileNames());
+ assertContents(file1Data, file1);
+ assertContents(file2Data, file2);
+
+ String name3 = "testFile3";
+
+ system.renameFile(name1, name3);
+
+ File file3 = system.getFile(name3);
+
+ assertEquals(new HashSet(Arrays.asList(name3, name2)), system.listFileNames());
+ assertContents(file1Data, file3);
+ assertContents(file2Data, file2);
+
+ system.deleteFile(name2);
+
+ assertEquals(new HashSet(Arrays.asList(name3)), system.listFileNames());
+
+ system.renameFile(name3, name2);
+
+ assertEquals(new HashSet(Arrays.asList(name2)), system.listFileNames());
+
+ file2 = system.getFile(name2);
+ assertContents(file1Data, file2);
+ }
+
+ private void assertContents(byte[] data, File file) throws IOException {
+ assertEquals(data.length, file.getLength());
+ InputStream is = file.getInputStream();
+ byte[] results = new byte[data.length];
+ assertEquals(file.getLength(), is.read(results));
+ assertEquals(-1, is.read());
+ is.close();
+
+ assertArrayEquals(data, results);
+ }
+
+ private byte[] writeRandomBytes(File file) throws IOException {
+ byte[] file1Data = getRandomBytes();
+ OutputStream outputStream = file.getOutputStream();
+ outputStream.write(file1Data);
+ outputStream.close();
+ return file1Data;
+ }
+
+ public byte[] getRandomBytes() {
+ byte[] data = new byte[rand.nextInt(LARGE_CHUNK) + SMALL_CHUNK];
+ rand.nextBytes(data);
+
+ return data;
+ }
+
+}