You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2017/08/04 12:00:01 UTC
svn commit: r1804106 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar:
TarConstants.java TarEntry.java TarEntryVisitor.java TarWriter.java
index/IndexWriter.java
Author: frm
Date: Fri Aug 4 12:00:01 2017
New Revision: 1804106
URL: http://svn.apache.org/viewvc?rev=1804106&view=rev
Log:
OAK-6518 - Encapsulate index writing logic in its own class
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java (with props)
Removed:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntryVisitor.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java?rev=1804106&r1=1804105&r2=1804106&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java Fri Aug 4 12:00:01 2017
@@ -26,20 +26,6 @@ class TarConstants {
static final String FILE_NAME_FORMAT = "data%05d%s.tar";
/**
- * Magic byte sequence at the end of the index block.
- * <p>
- * <ul>
- * <li>For each segment in that file, an index entry that contains the UUID,
- * the offset within the file and the size of the segment. Sorted by UUID,
- * to allow using interpolation search.</li>
- * <li>
- * The index footer, which contains metadata of the index (the size,
- * checksum).</li>
- * </ul>
- */
- static final int INDEX_MAGIC = ('\n' << 24) + ('1' << 16) + ('K' << 8) + '\n';
-
- /**
* Magic byte sequence at the end of the graph block.
* <p>
* The file is read from the end (the tar file is read from the end: the
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java?rev=1804106&r1=1804105&r2=1804106&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java Fri Aug 4 12:00:01 2017
@@ -42,23 +42,6 @@ class TarEntry {
}
};
- static final Comparator<TarEntry> IDENTIFIER_ORDER = new Comparator<TarEntry>() {
- @Override
- public int compare(TarEntry a, TarEntry b) {
- if (a.msb > b.msb) {
- return 1;
- } else if (a.msb < b.msb) {
- return -1;
- } else if (a.lsb > b.lsb) {
- return 1;
- } else if (a.lsb < b.lsb) {
- return -1;
- } else {
- return 0;
- }
- }
- };
-
private final long msb;
private final long lsb;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java?rev=1804106&r1=1804105&r2=1804106&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java Fri Aug 4 12:00:01 2017
@@ -31,7 +31,6 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.FILE_NAME_FORMAT;
import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC;
-import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.INDEX_MAGIC;
import java.io.Closeable;
import java.io.File;
@@ -40,7 +39,6 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
-import java.util.Arrays;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -50,6 +48,7 @@ import java.util.zip.CRC32;
import com.google.common.base.Charsets;
import com.google.common.base.Stopwatch;
+import org.apache.jackrabbit.oak.segment.file.tar.index.IndexWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -492,38 +491,23 @@ class TarWriter implements Closeable {
}
private void writeIndex() throws IOException {
- int indexSize = index.size() * TarEntry.SIZE + 16;
- int padding = getPaddingSize(indexSize);
+ IndexWriter writer = IndexWriter.newIndexWriter(BLOCK_SIZE);
- String indexName = file.getName() + ".idx";
- byte[] header = newEntryHeader(indexName, indexSize + padding);
-
- ByteBuffer buffer = ByteBuffer.allocate(indexSize);
- TarEntry[] sorted = index.values().toArray(new TarEntry[index.size()]);
- Arrays.sort(sorted, TarEntry.IDENTIFIER_ORDER);
- for (TarEntry entry : sorted) {
- buffer.putLong(entry.msb());
- buffer.putLong(entry.lsb());
- buffer.putInt(entry.offset());
- buffer.putInt(entry.size());
- buffer.putInt(entry.generation().getFull());
- buffer.putInt(entry.generation().getTail());
- buffer.put((byte) (entry.generation().isTail() ? 1 : 0));
- }
-
- CRC32 checksum = new CRC32();
- checksum.update(buffer.array(), 0, buffer.position());
- buffer.putInt((int) checksum.getValue());
- buffer.putInt(index.size());
- buffer.putInt(padding + indexSize);
- buffer.putInt(INDEX_MAGIC);
-
- access.write(header);
- if (padding > 0) {
- // padding comes *before* the index!
- access.write(ZERO_BYTES, 0, padding);
- }
- access.write(buffer.array());
+ for (TarEntry entry : index.values()) {
+ writer.addEntry(
+ entry.msb(),
+ entry.lsb(),
+ entry.offset(),
+ entry.size(),
+ entry.generation().getFull(),
+ entry.generation().getTail(),
+ entry.generation().isTail()
+ );
+ }
+
+ byte[] index = writer.write();
+ access.write(newEntryHeader(file.getName() + ".idx", index.length));
+ access.write(index);
}
private static byte[] newEntryHeader(String name, int size) {
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java?rev=1804106&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java Fri Aug 4 12:00:01 2017
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file.tar.index;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.zip.CRC32;
+
+public class IndexWriter {
+
+ private static class Entry {
+
+ long msb;
+
+ long lsb;
+
+ int offset;
+
+ int size;
+
+ int fullGeneration;
+
+ int tailGeneration;
+
+ boolean isTail;
+
+ }
+
+ public static IndexWriter newIndexWriter(int blockSize) {
+ checkArgument(blockSize > 0, "Invalid block size");
+ return new IndexWriter(blockSize);
+ }
+
+ private final int blockSize;
+
+ private final List<Entry> entries = new ArrayList<>();
+
+ private IndexWriter(int blockSize) {
+ this.blockSize = blockSize;
+ }
+
+ public void addEntry(long msb, long lsb, int offset, int size, int fullGeneration, int tailGeneration, boolean isTail) {
+ Entry entry = new Entry();
+ entry.msb = msb;
+ entry.lsb = lsb;
+ entry.offset = offset;
+ entry.size = size;
+ entry.fullGeneration = fullGeneration;
+ entry.tailGeneration = tailGeneration;
+ entry.isTail = isTail;
+ entries.add(entry);
+ }
+
+ public byte[] write() {
+ int dataSize = entries.size() * IndexEntryV2.SIZE + IndexV2.FOOTER_SIZE;
+ int totalSize = ((dataSize + blockSize - 1) / blockSize) * blockSize;
+
+ ByteBuffer buffer = ByteBuffer.allocate(totalSize);
+ buffer.position(totalSize - dataSize);
+
+ entries.sort((a, b) -> {
+ if (a.msb < b.msb) {
+ return -1;
+ }
+ if (a.msb > b.msb) {
+ return 1;
+ }
+ if (a.lsb < b.lsb) {
+ return -1;
+ }
+ if (a.lsb > b.lsb) {
+ return 1;
+ }
+ return 0;
+ });
+
+ for (Entry entry : entries) {
+ buffer.putLong(entry.msb);
+ buffer.putLong(entry.lsb);
+ buffer.putInt(entry.offset);
+ buffer.putInt(entry.size);
+ buffer.putInt(entry.fullGeneration);
+ buffer.putInt(entry.tailGeneration);
+ buffer.put((byte) (entry.isTail ? 1 : 0));
+ }
+
+ CRC32 checksum = new CRC32();
+ checksum.update(buffer.array(), totalSize - dataSize, dataSize - IndexV2.FOOTER_SIZE);
+
+ buffer.putInt((int) checksum.getValue());
+ buffer.putInt(entries.size());
+ buffer.putInt(totalSize);
+ buffer.putInt(IndexLoaderV2.MAGIC);
+
+ return buffer.array();
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/index/IndexWriter.java
------------------------------------------------------------------------------
svn:eol-style = native