You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2017/08/04 09:23:15 UTC
svn commit: r1804083 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index:
Index.java IndexEntry.java IndexEntryV2.java IndexLoader.java
IndexLoaderV2.java IndexV2.java
Author: frm
Date: Fri Aug 4 09:23:15 2017
New Revision: 1804083
URL: http://svn.apache.org/viewvc?rev=1804083&view=rev
Log:
OAK-6518 - Move the new index implementation in its own set of classes
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntryV2.java
- copied, changed from r1804082, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexV2.java
- copied, changed from r1804082, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java?rev=1804083&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java Fri Aug 4 09:23:15 2017
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+import java.util.Set;
+import java.util.UUID;
+
+public interface Index {
+
+ Set<UUID> getUUIDs();
+
+ IndexEntry findEntry(long msb, long lsb);
+
+ int size();
+
+ int entryCount();
+
+ IndexEntry entry(int i);
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java?rev=1804083&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java Fri Aug 4 09:23:15 2017
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+public interface IndexEntry {
+
+ long getMsb();
+
+ long getLsb();
+
+ int getPosition();
+
+ int getLength();
+
+ int getFullGeneration();
+
+ int getTailGeneration();
+
+ boolean isTail();
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntryV2.java (from r1804082, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntryV2.java?p2=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntryV2.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java&r1=1804082&r2=1804083&rev=1804083&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntry.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexEntryV2.java Fri Aug 4 09:23:15 2017
@@ -19,41 +19,50 @@ package org.apache.jackrabbit.oak.segmen
import java.nio.ByteBuffer;
-public class IndexEntry {
+class IndexEntryV2 implements IndexEntry {
+
+ static final int SIZE = 33;
private final ByteBuffer index;
private final int position;
- IndexEntry(ByteBuffer index, int position) {
+ IndexEntryV2(ByteBuffer index, int position) {
this.index = index;
this.position = position;
}
+ @Override
public long getMsb() {
return index.getLong(position);
}
+ @Override
public long getLsb() {
return index.getLong(position + 8);
}
+ @Override
public int getPosition() {
return index.getInt(position + 16);
}
+ @Override
public int getLength() {
return index.getInt(position + 20);
}
+ @Override
public int getFullGeneration() {
return index.getInt(position + 24);
}
+ @Override
public int getTailGeneration() {
return index.getInt(position + 28);
}
+ @Override
public boolean isTail() {
return index.get(position + 32) != 0;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java?rev=1804083&r1=1804082&r2=1804083&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoader.java Fri Aug 4 09:23:15 2017
@@ -27,12 +27,6 @@ import java.util.zip.CRC32;
public class IndexLoader {
- private static final int MAGIC = ('\n' << 24) + ('0' << 16) + ('K' << 8) + '\n';
-
- private static final int FOOTER_SIZE = 16;
-
- static final int ENTRY_SIZE = 33;
-
public static IndexLoader newIndexLoader(int blockSize) {
checkArgument(blockSize > 0, "Invalid block size");
return new IndexLoader(blockSize);
@@ -40,8 +34,11 @@ public class IndexLoader {
private final int blockSize;
+ private final IndexLoaderV2 v2;
+
private IndexLoader(int blockSize) {
this.blockSize = blockSize;
+ this.v2 = new IndexLoaderV2(blockSize);
}
public Index loadIndex(RandomAccessFile file) throws IOException, InvalidIndexException {
@@ -51,69 +48,16 @@ public class IndexLoader {
throw new InvalidIndexException(String.format("Unexpected size %d", length));
}
- // read the index metadata just before the two final zero blocks
- ByteBuffer meta = ByteBuffer.allocate(FOOTER_SIZE);
- file.seek(length - 2 * blockSize - FOOTER_SIZE);
- file.readFully(meta.array());
- int crc32 = meta.getInt();
- int count = meta.getInt();
- int bytes = meta.getInt();
- int magic = meta.getInt();
-
- if (magic != MAGIC) {
- throw new InvalidIndexException("Magic number mismatch");
- }
-
- if (count < 1 || bytes < count * ENTRY_SIZE + FOOTER_SIZE || bytes % blockSize != 0) {
- throw new InvalidIndexException("Invalid metadata");
- }
-
- // this involves seeking backwards in the file, which might not
- // perform well, but that's OK since we only do this once per file
- ByteBuffer index = ByteBuffer.allocate(count * ENTRY_SIZE);
- file.seek(length - 2 * blockSize - FOOTER_SIZE - count * ENTRY_SIZE);
- file.readFully(index.array());
- index.mark();
-
- CRC32 checksum = new CRC32();
- checksum.update(index.array());
- if (crc32 != (int) checksum.getValue()) {
- throw new InvalidIndexException("Invalid checksum");
- }
-
- long limit = length - 2 * blockSize - bytes - blockSize;
- long lastMsb = Long.MIN_VALUE;
- long lastLsb = Long.MIN_VALUE;
- byte[] entry = new byte[ENTRY_SIZE];
- for (int i = 0; i < count; i++) {
- index.get(entry);
-
- ByteBuffer buffer = wrap(entry);
- long msb = buffer.getLong();
- long lsb = buffer.getLong();
- int offset = buffer.getInt();
- int size = buffer.getInt();
-
- if (lastMsb > msb || (lastMsb == msb && lastLsb > lsb)) {
- throw new InvalidIndexException("Incorrect entry ordering");
- }
- if (lastMsb == msb && lastLsb == lsb && i > 0) {
- throw new InvalidIndexException("Duplicate entry");
- }
- if (offset < 0 || offset % blockSize != 0) {
- throw new InvalidIndexException("Invalid entry offset");
- }
- if (size < 1 || offset + size > limit) {
- throw new InvalidIndexException("Invalid entry size");
- }
+ ByteBuffer buffer = ByteBuffer.allocate(Integer.BYTES);
+ file.seek(length - 2 * blockSize - Integer.BYTES);
+ file.readFully(buffer.array());
+ int magic = buffer.getInt();
- lastMsb = msb;
- lastLsb = lsb;
+ if (magic == IndexLoaderV2.MAGIC) {
+ return v2.loadIndex(file);
}
- index.reset();
-
- return new Index(index);
+ throw new InvalidIndexException("Unrecognized magic number");
}
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java?rev=1804083&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java Fri Aug 4 09:23:15 2017
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+import static java.nio.ByteBuffer.wrap;
+
+import java.io.IOException;
+import java.io.RandomAccessFile;
+import java.nio.ByteBuffer;
+import java.util.zip.CRC32;
+
+class IndexLoaderV2 {
+
+ static final int MAGIC = ('\n' << 24) + ('0' << 16) + ('K' << 8) + '\n';
+
+ private final int blockSize;
+
+ IndexLoaderV2(int blockSize) {
+ this.blockSize = blockSize;
+ }
+
+ Index loadIndex(RandomAccessFile file) throws InvalidIndexException, IOException {
+ long length = file.length();
+ // read the index metadata just before the two final zero blocks
+ ByteBuffer meta = ByteBuffer.allocate(IndexV2.FOOTER_SIZE);
+ file.seek(length - 2 * blockSize - IndexV2.FOOTER_SIZE);
+ file.readFully(meta.array());
+ int crc32 = meta.getInt();
+ int count = meta.getInt();
+ int bytes = meta.getInt();
+ int magic = meta.getInt();
+
+ if (magic != MAGIC) {
+ throw new InvalidIndexException("Magic number mismatch");
+ }
+
+ if (count < 1 || bytes < count * IndexEntryV2.SIZE + IndexV2.FOOTER_SIZE || bytes % blockSize != 0) {
+ throw new InvalidIndexException("Invalid metadata");
+ }
+
+ // this involves seeking backwards in the file, which might not
+ // perform well, but that's OK since we only do this once per file
+ ByteBuffer index = ByteBuffer.allocate(count * IndexEntryV2.SIZE);
+ file.seek(length - 2 * blockSize - IndexV2.FOOTER_SIZE - count * IndexEntryV2.SIZE);
+ file.readFully(index.array());
+ index.mark();
+
+ CRC32 checksum = new CRC32();
+ checksum.update(index.array());
+ if (crc32 != (int) checksum.getValue()) {
+ throw new InvalidIndexException("Invalid checksum");
+ }
+
+ long limit = length - 2 * blockSize - bytes - blockSize;
+ long lastMsb = Long.MIN_VALUE;
+ long lastLsb = Long.MIN_VALUE;
+ byte[] entry = new byte[IndexEntryV2.SIZE];
+ for (int i = 0; i < count; i++) {
+ index.get(entry);
+
+ ByteBuffer buffer = wrap(entry);
+ long msb = buffer.getLong();
+ long lsb = buffer.getLong();
+ int offset = buffer.getInt();
+ int size = buffer.getInt();
+
+ if (lastMsb > msb || (lastMsb == msb && lastLsb > lsb)) {
+ throw new InvalidIndexException("Incorrect entry ordering");
+ }
+ if (lastMsb == msb && lastLsb == lsb && i > 0) {
+ throw new InvalidIndexException("Duplicate entry");
+ }
+ if (offset < 0 || offset % blockSize != 0) {
+ throw new InvalidIndexException("Invalid entry offset");
+ }
+ if (size < 1 || offset + size > limit) {
+ throw new InvalidIndexException("Invalid entry size");
+ }
+
+ lastMsb = msb;
+ lastLsb = lsb;
+ }
+
+ index.reset();
+
+ return new IndexV2(index);
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexLoaderV2.java
------------------------------------------------------------------------------
svn:eol-style = native
Copied: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexV2.java (from r1804082, jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexV2.java?p2=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexV2.java&p1=jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java&r1=1804082&r2=1804083&rev=1804083&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/Index.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexV2.java Fri Aug 4 09:23:15 2017
@@ -23,33 +23,37 @@ import java.nio.ByteBuffer;
import java.util.Set;
import java.util.UUID;
-public class Index {
+class IndexV2 implements Index {
+
+ static final int FOOTER_SIZE = 16;
private final ByteBuffer index;
- Index(ByteBuffer index) {
+ IndexV2(ByteBuffer index) {
this.index = index;
}
+ @Override
public Set<UUID> getUUIDs() {
- Set<UUID> uuids = newHashSetWithExpectedSize(index.remaining() / IndexLoader.ENTRY_SIZE);
+ Set<UUID> uuids = newHashSetWithExpectedSize(index.remaining() / IndexEntryV2.SIZE);
int position = index.position();
while (position < index.limit()) {
long msb = index.getLong(position);
long lsb = index.getLong(position + 8);
uuids.add(new UUID(msb, lsb));
- position += IndexLoader.ENTRY_SIZE;
+ position += IndexEntryV2.SIZE;
}
return uuids;
}
- public IndexEntry findEntry(long msb, long lsb) {
+ @Override
+ public IndexEntryV2 findEntry(long msb, long lsb) {
// The segment identifiers are randomly generated with uniform
// distribution, so we can use interpolation search to find the
// matching entry in the index. The average runtime is O(log log n).
int lowIndex = 0;
- int highIndex = index.remaining() / IndexLoader.ENTRY_SIZE - 1;
+ int highIndex = index.remaining() / IndexEntryV2.SIZE - 1;
float lowValue = Long.MIN_VALUE;
float highValue = Long.MAX_VALUE;
float targetValue = msb;
@@ -59,7 +63,7 @@ public class Index {
(highIndex - lowIndex)
* (targetValue - lowValue)
/ (highValue - lowValue));
- int position = index.position() + guessIndex * IndexLoader.ENTRY_SIZE;
+ int position = index.position() + guessIndex * IndexEntryV2.SIZE;
long m = index.getLong(position);
if (msb < m) {
highIndex = guessIndex - 1;
@@ -77,7 +81,7 @@ public class Index {
lowIndex = guessIndex + 1;
lowValue = m;
} else {
- return new IndexEntry(index, position);
+ return new IndexEntryV2(index, position);
}
}
}
@@ -85,16 +89,19 @@ public class Index {
return null;
}
+ @Override
public int size() {
- return index.remaining() + 16;
+ return index.remaining() + FOOTER_SIZE;
}
+ @Override
public int entryCount() {
- return index.remaining() / IndexLoader.ENTRY_SIZE;
+ return index.remaining() / IndexEntryV2.SIZE;
}
- public IndexEntry entry(int i) {
- return new IndexEntry(index, index.position() + i * IndexLoader.ENTRY_SIZE);
+ @Override
+ public IndexEntryV2 entry(int i) {
+ return new IndexEntryV2(index, index.position() + i * IndexEntryV2.SIZE);
}
}