You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2014/01/23 19:25:41 UTC
svn commit: r1560780 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: jukka
Date: Thu Jan 23 18:25:41 2014
New Revision: 1560780
URL: http://svn.apache.org/r1560780
Log:
OAK-1333: SegmentMK: Support for Blobs in external storage
Patch by Dominique Pfister
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java (with props)
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java
- copied, changed from r1560720, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/AbstractStore.java Thu Jan 23 18:25:41 2014
@@ -24,6 +24,7 @@ import java.util.Set;
import java.util.UUID;
import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
import org.apache.jackrabbit.oak.cache.CacheLIRS;
@@ -172,4 +173,9 @@ public abstract class AbstractStore impl
return type.isInstance(object) && ((Record) object).getStore() == this;
}
+ @Nullable
+ @Override
+ public ExternalBlob readBlob(String reference) {
+ return null;
+ }
}
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java?rev=1560780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java Thu Jan 23 18:25:41 2014
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.jackrabbit.oak.api.Blob;
+
+/**
+ * Marks a blob that is external.
+ */
+public interface ExternalBlob extends Blob {
+
+ /**
+ * Return a reference to this external blob.
+ *
+ * @return reference
+ */
+ public String getReference();
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlob.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Thu Jan 23 18:25:41 2014
@@ -440,6 +440,12 @@ public class Segment {
}
}
+ SegmentBlob createBlob(int offset) {
+ RecordId id = new RecordId(uuid, offset);
+ int n = readByte(offset) & 0xff;
+ return new SegmentBlob(this, id, (n & 0xe0) == 0xe0);
+ }
+
SegmentStream readStream(int offset) {
RecordId id = new RecordId(uuid, offset);
int pos = pos(offset, 1);
@@ -462,6 +468,25 @@ public class Segment {
}
}
+ String readBlobReference(int offset) {
+ int pos = pos(offset, 1);
+
+ int length = (data.get(pos++) & 0x1f) << 8
+ | (data.get(pos++) & 0xff);
+
+ byte[] bytes = new byte[length];
+ ByteBuffer buffer = data.duplicate();
+ buffer.position(pos + 8); // skip blob length
+ buffer.get(bytes);
+ return new String(bytes, Charsets.UTF_8);
+ }
+
+ long readBlobLength(int offset) {
+ long high = readInt(offset + 2);
+ long low = readInt(offset + 6);
+ return high << 32 | low;
+ }
+
//------------------------------------------------------------< Object >--
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java Thu Jan 23 18:25:41 2014
@@ -21,20 +21,34 @@ import javax.annotation.Nonnull;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import java.io.InputStream;
+
class SegmentBlob extends Record implements Blob {
- SegmentBlob(Segment segment, RecordId id) {
+ private boolean external;
+
+ SegmentBlob(Segment segment, RecordId id, boolean external) {
super(segment, id);
+
+ this.external = external;
}
@Override @Nonnull
- public SegmentStream getNewStream() {
+ public InputStream getNewStream() {
+ if (external) {
+ String refererence = getSegment().readBlobReference(getOffset());
+ return getStore().readBlob(refererence).getNewStream();
+ }
return getSegment().readStream(getOffset());
}
@Override
public long length() {
- SegmentStream stream = getNewStream();
+ if (external) {
+ return getSegment().readBlobLength(getOffset());
+ }
+
+ SegmentStream stream = (SegmentStream) getNewStream();
try {
return stream.getLength();
} finally {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentPropertyState.java Thu Jan 23 18:25:41 2014
@@ -162,7 +162,7 @@ class SegmentPropertyState extends Recor
@SuppressWarnings("unchecked")
private <T> T getValue(Segment segment, RecordId id, Type<T> type) {
if (type == BINARY) {
- return (T) new SegmentBlob(segment, id); // load binaries lazily
+ return (T) segment.createBlob(id.getOffset()); // load binaries lazily
}
String value = segment.readString(id);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentStore.java Thu Jan 23 18:25:41 2014
@@ -16,6 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import javax.annotation.Nullable;
import java.util.UUID;
public interface SegmentStore {
@@ -51,4 +52,11 @@ public interface SegmentStore {
*/
boolean isInstance(Object object, Class<? extends Record> type);
+ /**
+ * Read a blob from external storage.
+ *
+ * @param reference blob reference
+ * @return external blob
+ */
+ ExternalBlob readBlob(String reference);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Thu Jan 23 18:25:41 2014
@@ -487,6 +487,31 @@ public class SegmentWriter {
}
/**
+ * Write a reference to an external blob.
+ *
+ * @param reference reference
+ * @param blobLength blob length
+ * @return record id
+ */
+ private synchronized RecordId writeValueRecord(String reference, long blobLength) {
+ byte[] data = reference.getBytes(Charsets.UTF_8);
+ int length = data.length;
+
+ checkArgument(length < 8192);
+
+ RecordId id = prepare(RecordType.VALUE, 2 + 8 + length);
+ int len = length | 0xE000;
+ buffer[position++] = (byte) (len >> 8);
+ buffer[position++] = (byte) len;
+
+ writeLong(blobLength);
+
+ System.arraycopy(data, 0, buffer, position, length);
+ position += length;
+ return id;
+ }
+
+ /**
* Writes a block record containing the given block of bytes.
*
* @param bytes source buffer
@@ -643,13 +668,20 @@ public class SegmentWriter {
}
public SegmentBlob writeBlob(Blob blob) throws IOException {
- if (store.isInstance(blob, SegmentBlob.class)) {
+ if (blob instanceof ExternalBlob) {
+ return writeBlob((ExternalBlob) blob);
+ } else if (store.isInstance(blob, SegmentBlob.class)) {
return (SegmentBlob) blob;
} else {
return writeStream(blob.getNewStream());
}
}
+ private SegmentBlob writeBlob(ExternalBlob blob) {
+ RecordId id = writeValueRecord(blob.getReference(), blob.length());
+ return new SegmentBlob(dummySegment, id, true);
+ }
+
/**
* Writes a stream value record. The given stream is consumed
* <em>and closed</em> by this method.
@@ -669,7 +701,7 @@ public class SegmentWriter {
Closeables.close(stream, threw);
}
}
- return new SegmentBlob(dummySegment, id);
+ return new SegmentBlob(dummySegment, id, false);
}
private RecordId internalWriteStream(InputStream stream)
Copied: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java (from r1560720, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java?p2=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java&r1=1560720&r2=1560780&rev=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileBlob.java Thu Jan 23 18:25:41 2014
@@ -14,49 +14,58 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.oak.plugins.segment;
+package org.apache.jackrabbit.oak.plugins.segment.file;
+
+import org.apache.jackrabbit.oak.plugins.segment.ExternalBlob;
import javax.annotation.Nonnull;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
-import org.apache.jackrabbit.oak.api.Blob;
-import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+public class FileBlob implements ExternalBlob {
-class SegmentBlob extends Record implements Blob {
+ private final String path;
- SegmentBlob(Segment segment, RecordId id) {
- super(segment, id);
+ public FileBlob(String path) {
+ this.path = path;
}
- @Override @Nonnull
- public SegmentStream getNewStream() {
- return getSegment().readStream(getOffset());
+ public String getReference() {
+ return path;
}
+ @Nonnull
@Override
- public long length() {
- SegmentStream stream = getNewStream();
+ public InputStream getNewStream() {
try {
- return stream.getLength();
- } finally {
- stream.close();
+ return new FileInputStream(getFile());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
}
}
- //------------------------------------------------------------< Object >--
+ @Override
+ public long length() {
+ return getFile().length();
+ }
+
+ private File getFile() {
+ return new File(path);
+ }
@Override
- public boolean equals(Object object) {
- if (object == this || fastEquals(this, object)) {
- return true;
- } else {
- return object instanceof Blob
- && AbstractBlob.equal(this, (Blob) object);
+ public boolean equals(Object obj) {
+ if (obj instanceof FileBlob) {
+ FileBlob other = (FileBlob) obj;
+ return this.path.equals(other.path);
}
+ return super.equals(obj);
}
@Override
public int hashCode() {
- return 0;
+ return path.hashCode();
}
-
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1560780&r1=1560779&r2=1560780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Thu Jan 23 18:25:41 2014
@@ -38,6 +38,7 @@ import java.util.concurrent.atomic.Atomi
import javax.annotation.Nonnull;
import org.apache.jackrabbit.oak.plugins.segment.AbstractStore;
+import org.apache.jackrabbit.oak.plugins.segment.ExternalBlob;
import org.apache.jackrabbit.oak.plugins.segment.Journal;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
@@ -334,4 +335,9 @@ public class FileStore extends AbstractS
super.deleteSegment(segmentId);
}
+ @Override
+ public ExternalBlob readBlob(String reference) {
+ return new FileBlob(reference);
+ }
+
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java?rev=1560780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java Thu Jan 23 18:25:41 2014
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileBlob;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.After;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Random;
+
+import static junit.framework.Assert.assertTrue;
+
+public class ExternalBlobTest {
+
+ private SegmentStore store;
+ private SegmentNodeStore nodeStore;
+ private FileBlob fileBlob;
+
+ @Test
+ public void testCreateAndRead() throws Exception {
+ SegmentNodeStore nodeStore = getNodeStore();
+
+ NodeState state = nodeStore.getRoot().getChildNode("hello");
+ if (!state.exists()) {
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.child("hello");
+ nodeStore.merge(builder, EmptyHook.INSTANCE, null);
+ }
+
+ Blob blob = getFileBlob();
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.getChildNode("hello").setProperty("world", blob);
+ nodeStore.merge(builder, EmptyHook.INSTANCE, null);
+
+ state = nodeStore.getRoot().getChildNode("hello");
+ blob = state.getProperty("world").getValue(Type.BINARY);
+
+ assertTrue("Blob written and read must be equal",
+ AbstractBlob.equal(blob, getFileBlob()));
+ }
+
+ @After
+ public void close() {
+ if (store != null) {
+ store.close();
+ }
+ }
+
+ protected SegmentNodeStore getNodeStore() throws IOException {
+ if (nodeStore == null) {
+ store = new FileStore(new File("target", "ExternalBlobTest"), 256, false);
+ nodeStore = new SegmentNodeStore(store);
+ }
+ return nodeStore;
+ }
+
+ private FileBlob getFileBlob() throws IOException {
+ if (fileBlob == null) {
+ File file = File.createTempFile("blob", "tmp");
+ file.deleteOnExit();
+
+ byte[] data = new byte[2345];
+ new Random().nextBytes(data);
+ FileUtils.writeByteArrayToFile(file, data);
+
+ fileBlob = new FileBlob(file.getPath());
+ }
+ return fileBlob;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobTest.java
------------------------------------------------------------------------------
svn:eol-style = native