You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2015/10/27 12:20:22 UTC
svn commit: r1710780 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: mduerig
Date: Tue Oct 27 11:20:22 2015
New Revision: 1710780
URL: http://svn.apache.org/viewvc?rev=1710780&view=rev
Log:
OAK-3550: Add meta data to segments
Add meta data to segments in the format "{wid=W,sno=S,gc=G,t=T}"
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/PersistedCompactionMap.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/package-info.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java Tue Oct 27 11:20:22 2015
@@ -16,6 +16,18 @@
*/
package org.apache.jackrabbit.oak.plugins.segment;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.api.Type.BINARIES;
+import static org.apache.jackrabbit.oak.api.Type.BINARY;
+import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
@@ -38,18 +50,6 @@ import org.apache.jackrabbit.oak.spi.sta
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static com.google.common.collect.Lists.newArrayList;
-import static com.google.common.collect.Maps.newHashMap;
-import static org.apache.jackrabbit.oak.api.Type.BINARIES;
-import static org.apache.jackrabbit.oak.api.Type.BINARY;
-import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
-
/**
* Tool for compacting segments.
*/
@@ -115,7 +115,8 @@ public class Compactor {
}
public Compactor(FileStore store, CompactionStrategy compactionStrategy, Supplier<Boolean> cancel) {
- this.writer = store.createSegmentWriter();
+ String wid = "c-" + store.getTracker().getCompactionMap().getGeneration() + 1;
+ this.writer = store.createSegmentWriter(wid);
if (compactionStrategy.getPersistCompactionMap()) {
this.map = new PersistedCompactionMap(store);
} else {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/PersistedCompactionMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/PersistedCompactionMap.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/PersistedCompactionMap.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/PersistedCompactionMap.java Tue Oct 27 11:20:22 2015
@@ -193,7 +193,7 @@ public class PersistedCompactionMap impl
base = baseEntry == null ? null : new MapRecord(baseEntry.getValue());
if (writer == null) {
- writer = store.createSegmentWriter();
+ writer = store.createSegmentWriter(createWid());
}
Map<String, RecordId> offsetMap = newHashMap();
@@ -218,7 +218,7 @@ public class PersistedCompactionMap impl
if (!segmentIdMap.isEmpty()) {
if (writer == null) {
- writer = store.createSegmentWriter();
+ writer = store.createSegmentWriter(createWid());
}
RecordId previousBaseId = entries == null ? null : entries.getRecordId();
@@ -237,6 +237,11 @@ public class PersistedCompactionMap impl
}
}
+ @Nonnull
+ private String createWid() {
+ return "cm-" + store.getTracker().getCompactionMap().getGeneration() + 1;
+ }
+
/**
* @return 0
*/
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java Tue Oct 27 11:20:22 2015
@@ -24,6 +24,7 @@ import java.security.SecureRandom;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.CheckForNull;
@@ -106,6 +107,11 @@ public class SegmentTracker {
*/
private final CacheLIRS<SegmentId, Segment> segmentCache;
+ /**
+ * Number of segments
+ */
+ private final AtomicInteger segmentCounter = new AtomicInteger();
+
public SegmentTracker(SegmentStore store, int cacheSizeMB,
SegmentVersion version) {
for (int i = 0; i < tables.length; i++) {
@@ -115,7 +121,7 @@ public class SegmentTracker {
this.store = store;
this.compactionMap = new AtomicReference<CompactionMap>(
CompactionMap.EMPTY);
- this.writer = new SegmentWriter(store, this, version);
+ this.writer = new SegmentWriter(store, this, version, "sys");
StringCache c;
if (DISABLE_STRING_CACHE) {
c = null;
@@ -146,6 +152,14 @@ public class SegmentTracker {
this(store, DEFAULT_MEMORY_CACHE_SIZE, SegmentVersion.V_11);
}
+ /**
+ * Increment and get the number of segments
+ * @return
+ */
+ int getNextSegmentNo() {
+ return segmentCounter.incrementAndGet();
+ }
+
@Nonnull
public CacheStats getSegmentCacheStats() {
return new CacheStats(segmentCache, "Segment Cache", null, -1);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Tue Oct 27 11:20:22 2015
@@ -28,6 +28,8 @@ import static com.google.common.collect.
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newLinkedHashMap;
import static com.google.common.collect.Sets.newHashSet;
+import static java.lang.System.currentTimeMillis;
+import static java.lang.System.identityHashCode;
import static java.util.Arrays.asList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.nCopies;
@@ -54,6 +56,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import javax.jcr.PropertyType;
@@ -88,6 +91,8 @@ public class SegmentWriter {
static final int BLOCK_SIZE = 1 << 12; // 4kB
+ private static final AtomicInteger SEGMENT_COUNTER = new AtomicInteger();
+
static byte[] createNewBuffer(SegmentVersion v) {
byte[] buffer = new byte[Segment.MAX_SEGMENT_SIZE];
buffer[0] = '0';
@@ -162,12 +167,54 @@ public class SegmentWriter {
*/
private final SegmentVersion version;
+ /**
+ * Id of this writer.
+ */
+ private final String wid;
+
public SegmentWriter(SegmentStore store, SegmentTracker tracker, SegmentVersion version) {
+ this(store, tracker, version, null);
+ }
+
+ /**
+ * @param store store to write to
+ * @param tracker segment tracker for that {@code store}
+ * @param version segment version to write
+ * @param wid id of this writer
+ */
+ public SegmentWriter(SegmentStore store, SegmentTracker tracker, SegmentVersion version, String wid) {
this.store = store;
this.tracker = tracker;
this.version = version;
this.buffer = createNewBuffer(version);
+ this.wid = wid == null
+ ? "w-" + identityHashCode(this)
+ : wid;
+ newSegment(wid);
+ }
+
+ /**
+ * Allocate a new segment and write the segment meta data.
+ * The segment meta data is a string of the format {@code "{wid=W,sno=S,gc=G,t=T}"}
+ * where:
+ * <ul>
+ * <li>{@code W} is the writer id {@code wid}, </li>
+ * <li>{@code S} is a unique, increasing sequence number corresponding to the allocation order
+ * of the segments in this store, </li>
+ * <li>{@code G} is the garbage collection generation (i.e. the number of compaction cycles
+ * that have been run),</li>
+ * <li>{@code T} is a time stamp according to {@link System#currentTimeMillis()}.</li>
+ * </ul>
+ * The segment meta data is guaranteed to be the first string record in a segment.
+ * @param wid the writer id
+ */
+ private void newSegment(String wid) {
this.segment = new Segment(tracker, buffer);
+ writeString(
+ "{wid=" + wid +
+ ",sno=" + tracker.getNextSegmentNo() +
+ ",gc=" + tracker.getCompactionMap().getGeneration() +
+ ",t=" + currentTimeMillis() + "}");
}
/**
@@ -244,7 +291,7 @@ public class SegmentWriter {
blobrefs.clear();
length = 0;
position = buffer.length;
- segment = new Segment(tracker, buffer);
+ newSegment(wid);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Tue Oct 27 11:20:22 2015
@@ -893,8 +893,8 @@ public class FileStore implements Segmen
/**
* @return a new {@link SegmentWriter} instance for writing to this store.
*/
- public SegmentWriter createSegmentWriter() {
- return new SegmentWriter(this, tracker, getVersion());
+ public SegmentWriter createSegmentWriter(String wid) {
+ return new SegmentWriter(this, tracker, getVersion(), wid);
}
/**
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/package-info.java Tue Oct 27 11:20:22 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("2.2.0")
+@Version("3.0.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.segment.file;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/package-info.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/package-info.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/package-info.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/package-info.java Tue Oct 27 11:20:22 2015
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@Version("5.1.0")
+@Version("5.2.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.plugins.segment;
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java?rev=1710780&r1=1710779&r2=1710780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentSizeTest.java Tue Oct 27 11:20:22 2015
@@ -40,29 +40,29 @@ public class SegmentSizeTest {
@Test
public void testNodeSize() {
NodeBuilder builder = EMPTY_NODE.builder();
- assertEquals(48, getSize(builder));
+ assertEquals(96, getSize(builder));
assertEquals(4, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
- assertEquals(48, getSize(builder));
+ assertEquals(96, getSize(builder));
assertEquals(8, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.setProperty("foo", "bar");
builder.setProperty("baz", 123);
- assertEquals(80, getSize(builder));
+ assertEquals(128, getSize(builder));
assertEquals(16, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.child("foo");
- assertEquals(64, getSize(builder));
+ assertEquals(112, getSize(builder));
assertEquals(12, getAmortizedSize(builder));
builder = EMPTY_NODE.builder();
builder.child("foo");
builder.child("bar");
- assertEquals(96, getSize(builder));
+ assertEquals(144, getSize(builder));
assertEquals(40, getAmortizedSize(builder));
}
@@ -118,7 +118,7 @@ public class SegmentSizeTest {
public void testAccessControlNodes() {
NodeBuilder builder = EMPTY_NODE.builder();
builder.setProperty("jcr:primaryType", "rep:ACL", Type.NAME);
- assertEquals(48, getSize(builder));
+ assertEquals(96, getSize(builder));
assertEquals(4, getAmortizedSize(builder));
NodeBuilder deny = builder.child("deny");
@@ -126,7 +126,7 @@ public class SegmentSizeTest {
deny.setProperty("rep:principalName", "everyone");
deny.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(176, getSize(builder));
+ assertEquals(224, getSize(builder));
assertEquals(32, getAmortizedSize(builder));
NodeBuilder allow = builder.child("allow");
@@ -134,7 +134,7 @@ public class SegmentSizeTest {
allow.setProperty("rep:principalName", "administrators");
allow.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(320, getSize(builder));
+ assertEquals(352, getSize(builder));
assertEquals(84, getAmortizedSize(builder));
NodeBuilder deny0 = builder.child("deny0");
@@ -143,7 +143,7 @@ public class SegmentSizeTest {
deny0.setProperty("rep:glob", "*/activities/*");
builder.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:read"), Type.NAMES));
- assertEquals(416, getSize(builder));
+ assertEquals(464, getSize(builder));
assertEquals(124, getAmortizedSize(builder));
NodeBuilder allow0 = builder.child("allow0");
@@ -151,7 +151,7 @@ public class SegmentSizeTest {
allow0.setProperty("rep:principalName", "user-administrators");
allow0.setProperty(PropertyStates.createProperty(
"rep:privileges", ImmutableList.of("jcr:all"), Type.NAMES));
- assertEquals(480, getSize(builder));
+ assertEquals(528, getSize(builder));
assertEquals(160, getAmortizedSize(builder));
}
@@ -168,7 +168,7 @@ public class SegmentSizeTest {
SegmentNodeState state = writer.writeNode(builder.getNodeState());
writer.flush();
Segment segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(27520, segment.size());
+ assertEquals(27568, segment.size());
writer.flush(); // force flushing of the previous segment
@@ -177,7 +177,7 @@ public class SegmentSizeTest {
state = writer.writeNode(builder.getNodeState());
writer.flush();
segment = store.readSegment(state.getRecordId().getSegmentId());
- assertEquals(496, segment.size());
+ assertEquals(544, segment.size());
}
private int getSize(NodeBuilder builder) {