You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2013/09/16 22:07:58 UTC
svn commit: r1523784 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
test/java/org/apache/jackrabbit/oak/plugins/segment/file/
Author: jukka
Date: Mon Sep 16 20:07:58 2013
New Revision: 1523784
URL: http://svn.apache.org/r1523784
Log:
OAK-1001: SegmentMK: 32bit support for the file backend
Add write functionality to the 32bit code, plus a test case
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java
- copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java
- copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java
- copied, changed from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
Removed:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java (from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileAccess.java Mon Sep 16 20:07:58 2013
@@ -16,37 +16,20 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.file;
-import static java.nio.channels.FileChannel.MapMode.READ_WRITE;
-
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
-class MappedTarFile extends TarFile {
+interface FileAccess {
+
+ int length() throws IOException;
+
+ ByteBuffer read(int position, int length) throws IOException;
+
+ void write(int position, byte[] b, int offset, int length)
+ throws IOException;
- private final ByteBuffer buffer;
+ void flush() throws IOException;
- MappedTarFile(File file) throws IOException {
- RandomAccessFile f = new RandomAccessFile(file, "rw");
- try {
- buffer = f.getChannel().map(READ_WRITE, 0, f.length());
- } finally {
- f.close();
- }
- }
-
- @Override
- protected int length() {
- return buffer.limit();
- }
-
- @Override
- protected ByteBuffer read(int position, int length) {
- ByteBuffer entry = buffer.asReadOnlyBuffer();
- entry.position(position);
- entry.limit(position + length);
- return entry;
- }
+ void close() throws IOException;
-}
+}
\ No newline at end of file
Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java (from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedTarFile.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/MappedAccess.java Mon Sep 16 20:07:58 2013
@@ -22,31 +22,53 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
-class MappedTarFile extends TarFile {
+class MappedAccess implements FileAccess {
- private final ByteBuffer buffer;
+ private final MappedByteBuffer buffer;
- MappedTarFile(File file) throws IOException {
+ MappedAccess(File file, int length) throws IOException {
RandomAccessFile f = new RandomAccessFile(file, "rw");
try {
- buffer = f.getChannel().map(READ_WRITE, 0, f.length());
+ long l = f.length();
+ if (l == 0) { // it's a new file
+ l = length;
+ }
+ buffer = f.getChannel().map(READ_WRITE, 0, l);
} finally {
f.close();
}
}
@Override
- protected int length() {
+ public int length() {
return buffer.limit();
}
@Override
- protected ByteBuffer read(int position, int length) {
+ public ByteBuffer read(int position, int length) {
ByteBuffer entry = buffer.asReadOnlyBuffer();
entry.position(position);
entry.limit(position + length);
- return entry;
+ return entry.slice();
+ }
+
+ @Override
+ public void write(int position, byte[] b, int offset, int length)
+ throws IOException {
+ ByteBuffer entry = buffer.duplicate();
+ entry.position(position);
+ entry.put(b, offset, length);
+ }
+
+ @Override
+ public void flush() {
+ buffer.force();
+ }
+
+ @Override
+ public void close() {
}
}
Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java (from r1523692, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java&r1=1523692&r2=1523784&rev=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccessTarFile.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/RandomAccess.java Mon Sep 16 20:07:58 2013
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.file;
+import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import java.io.File;
@@ -23,16 +24,25 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
-class RandomAccessTarFile extends TarFile {
+import javax.annotation.Nonnull;
+
+class RandomAccess implements FileAccess {
private final RandomAccessFile file;
- RandomAccessTarFile(File file) throws IOException {
- this.file = new RandomAccessFile(file, "rw");
+ RandomAccess(@Nonnull File file) throws IOException {
+ this.file = new RandomAccessFile(checkNotNull(file), "rw");
+ }
+
+ @Override
+ public int length() throws IOException {
+ long length = file.length();
+ checkState(length < Integer.MAX_VALUE);
+ return (int) length;
}
@Override
- protected synchronized ByteBuffer read(int position, int length)
+ public synchronized ByteBuffer read(int position, int length)
throws IOException {
ByteBuffer entry = ByteBuffer.allocate(length);
file.seek(position);
@@ -41,14 +51,20 @@ class RandomAccessTarFile extends TarFil
}
@Override
- protected int length() throws IOException {
- long length = file.length();
- checkState(length < Integer.MAX_VALUE);
- return (int) length;
+ public synchronized void write(
+ int position, byte[] buffer, int offset, int length)
+ throws IOException {
+ file.seek(position);
+ file.write(buffer, offset, length);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ file.getFD().sync();
}
@Override
- void close() throws IOException {
+ public void close() throws IOException {
file.close();
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java?rev=1523784&r1=1523783&r2=1523784&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFile.java Mon Sep 16 20:07:58 2013
@@ -16,44 +16,201 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.file;
-import static java.util.Collections.emptyMap;
+import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.base.Preconditions.checkState;
+import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Map;
import java.util.UUID;
-abstract class TarFile {
+import com.google.common.collect.ImmutableMap;
- protected static class Entry {
+class TarFile {
- int position;
+ private static boolean USE_MEMORY_MAPPING =
+ System.getProperty("tarmk.mmap") != null
+ ? Boolean.getBoolean("tarmk.mmap")
+ : "64".equals(System.getProperty("sun.arch.data.model"));
- int length;
+ /** The tar file block size. */
+ private static final int BLOCK_SIZE = 512;
- Entry(int position, int length) {
- this.position = position;
- this.length = length;
+ private static final byte[] ZERO_BYTES = new byte[BLOCK_SIZE];
+
+ private static class Location {
+
+ int offset;
+
+ int size;
+
+ Location(int offset, int size) {
+ this.offset = offset;
+ this.size = size;
}
}
- private volatile Map<UUID, Entry> entries = emptyMap();
+ private final FileAccess file;
+
+ private int position = 0;
+
+ private final int maxLength;
+
+ private volatile Map<UUID, Location> entries;
+
+ TarFile(File file, int maxLength) throws IOException {
+ long len = file.length();
+ checkState(len <= Integer.MAX_VALUE);
+ this.maxLength = Math.max((int) len, maxLength);
+
+ if (USE_MEMORY_MAPPING) {
+ this.file = new MappedAccess(file, maxLength);
+ } else {
+ this.file = new RandomAccess(file);
+ }
+
+ ImmutableMap.Builder<UUID, Location> builder = ImmutableMap.builder();
+
+ this.position = 0;
+ while (position + BLOCK_SIZE <= len) {
+ // read the tar header block
+ ByteBuffer buffer = this.file.read(position, BLOCK_SIZE);
+ String name = readString(buffer, 100);
+ buffer.position(124);
+ int size = readNumber(buffer, 12);
+ // TODO: verify the checksum, magic, etc.?
+
+ if (name.isEmpty() && size == 0) {
+ break; // no more entries in this file
+ }
+
+ try {
+ UUID id = UUID.fromString(name);
+ builder.put(id, new Location(position + BLOCK_SIZE, size));
+ } catch (IllegalArgumentException e) {
+ throw new IOException("Unexpected tar entry: " + name);
+ }
+
+ position += (1 + (size + BLOCK_SIZE - 1) / BLOCK_SIZE) * BLOCK_SIZE;
+ }
+
+ this.entries = builder.build();
+ }
ByteBuffer readEntry(UUID id) throws IOException {
- Entry entry = entries.get(id);
- if (entry != null) {
- return read(entry.position, entry.length);
+ Location location = entries.get(id);
+ if (location != null) {
+ return file.read(location.offset, location.size);
} else {
return null;
}
}
- protected abstract int length() throws IOException;
+ synchronized boolean writeEntry(UUID id, byte[] b, int offset, int size)
+ throws IOException {
+ if (position + BLOCK_SIZE + size > maxLength) {
+ return false;
+ }
+
+ byte[] header = new byte[BLOCK_SIZE];
- protected abstract ByteBuffer read(int position, int length)
- throws IOException;
+ // File name
+ byte[] n = id.toString().getBytes(UTF_8);
+ System.arraycopy(n, 0, header, 0, n.length);
+
+ // File mode
+ System.arraycopy(
+ String.format("%07o", 0400).getBytes(UTF_8), 0,
+ header, 100, 7);
+
+ // User's numeric user ID
+ System.arraycopy(
+ String.format("%07o", 0).getBytes(UTF_8), 0,
+ header, 108, 7);
+
+ // Group's numeric user ID
+ System.arraycopy(
+ String.format("%07o", 0).getBytes(UTF_8), 0,
+ header, 116, 7);
+
+ // File size in bytes (octal basis)
+ System.arraycopy(
+ String.format("%011o", size).getBytes(UTF_8), 0,
+ header, 124, 11);
+
+ // Last modification time in numeric Unix time format (octal)
+ long time = System.currentTimeMillis() / 1000;
+ System.arraycopy(
+ String.format("%011o", time).getBytes(UTF_8), 0,
+ header, 136, 11);
+
+ // Checksum for header record
+ System.arraycopy(
+ new byte[] { ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ' }, 0,
+ header, 148, 8);
+
+ // Type flag
+ header[156] = '0';
+
+ // Compute checksum
+ int checksum = 0;
+ for (int i = 0; i < header.length; i++) {
+ checksum += header[i] & 0xff;
+ }
+ System.arraycopy(
+ String.format("%06o", checksum).getBytes(UTF_8), 0,
+ header, 148, 6);
+ header[154] = 0;
+
+ file.write(position, header, 0, BLOCK_SIZE);
+ position += BLOCK_SIZE;
+
+ file.write(position, b, offset, size);
+ entries = ImmutableMap.<UUID, Location>builder()
+ .putAll(entries)
+ .put(id, new Location(position, size))
+ .build();
+ position += size;
+
+ int padding = BLOCK_SIZE - position % BLOCK_SIZE;
+ if (padding < BLOCK_SIZE) {
+ file.write(position, ZERO_BYTES, 0, padding);
+ position += padding;
+ }
+
+ return true;
+ }
void close() throws IOException {
+ file.flush();
+ file.close();
}
+
+ private static String readString(ByteBuffer buffer, int fieldSize) {
+ byte[] b = new byte[fieldSize];
+ buffer.get(b);
+ int n = 0;
+ while (n < fieldSize && b[n] != 0) {
+ n++;
+ }
+ return new String(b, 0, n, UTF_8);
+ }
+
+ private static int readNumber(ByteBuffer buffer, int fieldSize) {
+ byte[] b = new byte[fieldSize];
+ buffer.get(b);
+ int number = 0;
+ for (int i = 0; i < fieldSize; i++) {
+ int digit = b[i] & 0xff;
+ if ('0' <= digit && digit <= '7') {
+ number = number * 8 + digit - '0';
+ } else {
+ break;
+ }
+ }
+ return number;
+ }
+
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java?rev=1523784&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/TarFileTest.java Mon Sep 16 20:07:58 2013
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static junit.framework.Assert.assertEquals;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TarFileTest {
+
+ private File file;
+
+ @Before
+ public void setUp() throws IOException {
+ file = File.createTempFile("TarFileTest", ".tar");
+ }
+
+ @After
+ public void tearDown() {
+ // file.delete();
+ }
+
+ @Test
+ public void testOpenClose() throws IOException {
+ TarFile tar = new TarFile(file, 1024);
+ tar.close();
+ }
+
+ @Test
+ public void testWriteAndRead() throws IOException {
+ UUID id = UUID.randomUUID();
+ byte[] data = "Hello, World!".getBytes(UTF_8);
+
+ TarFile tar = new TarFile(file, 1024);
+ try {
+ tar.writeEntry(id, data, 0, data.length);
+ assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
+ } finally {
+ tar.close();
+ }
+
+ assertEquals(1024, file.length());
+
+ tar = new TarFile(file, 1024);
+ try {
+ assertEquals(ByteBuffer.wrap(data), tar.readEntry(id));
+ } finally {
+ tar.close();
+ }
+ }
+
+}