You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2015/05/06 09:15:16 UTC
svn commit: r1677939 - in /jackrabbit/oak/trunk/oak-tarmk-standby/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/
main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/
test/java/org/apache/jackrabbit/oak/plugins/se...
Author: alexparvulescu
Date: Wed May 6 07:15:15 2015
New Revision: 1677939
URL: http://svn.apache.org/r1677939
Log:
OAK-2817 TARMK Cold Standby cleanup removes too many binary segments
Modified:
jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java
jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java
jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java
jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml
Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java Wed May 6 07:15:15 2015
@@ -25,8 +25,6 @@ import io.netty.channel.ChannelHandlerCo
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.util.concurrent.EventExecutorGroup;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
@@ -97,7 +95,7 @@ public class SegmentLoaderHandler extend
}
try {
- store.setLoader(this);
+ store.preSync(this);
SegmentNodeState before = store.getHead();
SegmentNodeBuilder builder = before.builder();
@@ -118,14 +116,7 @@ public class SegmentLoaderHandler extend
}
log.debug("did reread locally corrupt segment " + id + " with size " + s.size());
- ByteArrayOutputStream bout = new ByteArrayOutputStream(s.size());
- try {
- s.writeTo(bout);
- } catch (IOException f) {
- log.error("can't wrap segment to output stream", f);
- throw e;
- }
- store.writeSegment(s.getSegmentId(), bout.toByteArray(), 0, s.size());
+ store.persist(s.getSegmentId(), s);
}
} while(true);
boolean ok = store.setHead(before, builder.getNodeState());
@@ -144,6 +135,7 @@ public class SegmentLoaderHandler extend
}
}
} finally {
+ store.postSync();
close();
}
}
Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java Wed May 6 07:15:15 2015
@@ -16,11 +16,16 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.standby.store;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.ArrayDeque;
import java.util.Deque;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.jackrabbit.oak.api.Blob;
@@ -70,35 +75,88 @@ public class StandbyStore implements Seg
@Override
public Segment readSegment(SegmentId sid) {
+ callId++;
Deque<SegmentId> ids = new ArrayDeque<SegmentId>();
ids.offer(sid);
int err = 0;
- Set<SegmentId> seen = new HashSet<SegmentId>();
+ Set<SegmentId> persisted = new HashSet<SegmentId>();
+
+ Map<SegmentId, Segment> cache = new HashMap<SegmentId, Segment>();
+ long cacheOps = 0;
+
+ long cacheWeight = 0;
+ long maxWeight = 0;
+ long maxKeys = 0;
while (!ids.isEmpty()) {
SegmentId id = ids.remove();
- if (!seen.contains(id) && !delegate.containsSegment(id)) {
- log.debug("trying to read segment " + id);
- Segment s = loader.readSegment(id.toString());
+ if (!persisted.contains(id) && !delegate.containsSegment(id)) {
+ Segment s;
+ boolean logRefs = true;
+ if (cache.containsKey(id)) {
+ s = cache.remove(id);
+ cacheWeight -= s.size();
+ cacheOps++;
+ logRefs = false;
+ } else {
+ log.debug("transferring segment {}", id);
+ s = loader.readSegment(id.toString());
+ }
+
if (s != null) {
- log.debug("got segment " + id + " with size " + s.size());
- ByteArrayOutputStream bout = new ByteArrayOutputStream(
+ log.debug("processing segment {} with size {}", id,
s.size());
if (id.isDataSegmentId()) {
- ids.addAll(s.getReferencedIds());
+ boolean hasPendingRefs = false;
+ List<SegmentId> refs = s.getReferencedIds();
+ if (logRefs) {
+ log.debug("{} -> {}", id, refs);
+ }
+ for (SegmentId nr : refs) {
+ // skip already persisted or self-ref
+ if (persisted.contains(nr) || id.equals(nr)) {
+ continue;
+ }
+ hasPendingRefs = true;
+ if (!ids.contains(nr)) {
+ if (nr.isBulkSegmentId()) {
+ // binaries first
+ ids.addFirst(nr);
+ } else {
+ // data segments last
+ ids.add(nr);
+ }
+ }
+ }
+
+ if (!hasPendingRefs) {
+ persisted.add(id);
+ persist(id, s);
+ } else {
+ // persist it later, after the refs are in place
+ ids.add(id);
+
+ // TODO there is a chance this might introduce
+ // a OOME because of the position of the current
+ // segment in the processing queue. putting it at
+ // the end of the current queue means it will stay
+ // in the cache until the pending queue of the
+ // segment's references is processed.
+ cache.put(id, s);
+ cacheWeight += s.size();
+ cacheOps++;
+
+ maxWeight = Math.max(maxWeight, cacheWeight);
+ maxKeys = Math.max(maxKeys, cache.size());
+ }
+ } else {
+ persisted.add(id);
+ persist(id, s);
}
- try {
- s.writeTo(bout);
- writeSegment(id, bout.toByteArray(), 0, s.size());
- } catch (IOException e) {
- throw new IllegalStateException(
- "Unable to write remote segment " + id, e);
- }
- seen.add(id);
- ids.removeAll(seen);
+ ids.removeAll(persisted);
err = 0;
} else {
- log.error("could NOT read segment " + id);
+ log.error("could NOT read segment {}", id);
if (loader.isClosed() || err == 4) {
loader.close();
throw new IllegalStateException(
@@ -108,14 +166,44 @@ public class StandbyStore implements Seg
ids.addFirst(id);
}
} else {
- seen.add(id);
+ persisted.add(id);
}
}
-
- log.debug("calling delegate to return segment " + sid);
+ cacheStats.put(callId, "W: " + humanReadableByteCount(maxWeight)
+ + ", Keys: " + maxKeys + ", Ops: " + cacheOps);
return delegate.readSegment(sid);
}
+ public void persist(SegmentId in, Segment s) {
+ SegmentId id = delegate.getTracker().getSegmentId(
+ in.getMostSignificantBits(), in.getLeastSignificantBits());
+ log.debug("persisting segment {} with size {}", id, s.size());
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream(s.size());
+ s.writeTo(bout);
+ writeSegment(id, bout.toByteArray(), 0, s.size());
+ } catch (IOException e) {
+ throw new IllegalStateException("Unable to write remote segment "
+ + id, e);
+ }
+ }
+
+ private long callId = 0;
+ private Map<Long, String> cacheStats;
+
+ public void preSync(RemoteSegmentLoader loader) {
+ this.loader = loader;
+ this.cacheStats = new HashMap<Long, String>();
+ }
+
+ public void postSync() {
+ loader = null;
+ if (log.isDebugEnabled() && !cacheStats.isEmpty()) {
+ log.debug("sync cache stats {}", cacheStats);
+ }
+ cacheStats = null;
+ }
+
@Override
public void writeSegment(SegmentId id, byte[] bytes, int offset, int length) {
delegate.writeSegment(id, bytes, offset, length);
@@ -141,10 +229,6 @@ public class StandbyStore implements Seg
delegate.gc();
}
- public void setLoader(RemoteSegmentLoader loader) {
- this.loader = loader;
- }
-
public long size() {
if (delegate instanceof FileStore) {
try {
Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java Wed May 6 07:15:15 2015
@@ -18,14 +18,12 @@
*/
package org.apache.jackrabbit.oak.plugins.segment.standby;
-import static org.apache.jackrabbit.oak.plugins.segment.SegmentTestUtils.createTmpTargetDir;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
-import java.io.File;
import java.io.IOException;
import java.util.Random;
@@ -103,11 +101,13 @@ public class StandbyTestIT extends TestB
try {
- for (int i = 0; i < 10; i++) {
+ for (int i = 0; i < 5; i++) {
String cp = store.checkpoint(Long.MAX_VALUE);
cl.run();
assertEquals(primary.getHead(), secondary.getHead());
assertTrue(store.release(cp));
+ cl.cleanup();
+ assertTrue(secondary.size() > blobSize);
}
} finally {
Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml Wed May 6 07:15:15 2015
@@ -22,11 +22,18 @@
</encoder>
</appender>
+ <appender name="file" class="ch.qos.logback.core.FileAppender">
+ <file>target/unit-tests.log</file>
+ <encoder>
+ <pattern>%date{HH:mm:ss.SSS} %-5level %-40([%thread] %F:%L) %msg%n</pattern>
+ </encoder>
+ </appender>
+
<logger name="org.apache.jackrabbit.oak.plugins.segment.standby" level="ERROR"/>
<logger name="org.apache.jackrabbit.oak.plugins.blob.datastore" level="ERROR"/>
<root level="INFO">
- <appender-ref ref="console"/>
+ <appender-ref ref="file"/>
</root>
</configuration>