You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2015/05/06 09:15:16 UTC

svn commit: r1677939 - in /jackrabbit/oak/trunk/oak-tarmk-standby/src: main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/ main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/ test/java/org/apache/jackrabbit/oak/plugins/se...

Author: alexparvulescu
Date: Wed May  6 07:15:15 2015
New Revision: 1677939

URL: http://svn.apache.org/r1677939
Log:
OAK-2817 TARMK Cold Standby cleanup removes too many binary segments


Modified:
    jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java
    jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java
    jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java
    jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml

Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/client/SegmentLoaderHandler.java Wed May  6 07:15:15 2015
@@ -25,8 +25,6 @@ import io.netty.channel.ChannelHandlerCo
 import io.netty.channel.ChannelInboundHandlerAdapter;
 import io.netty.util.concurrent.EventExecutorGroup;
 
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.TimeUnit;
@@ -97,7 +95,7 @@ public class SegmentLoaderHandler extend
         }
 
         try {
-            store.setLoader(this);
+            store.preSync(this);
             SegmentNodeState before = store.getHead();
             SegmentNodeBuilder builder = before.builder();
 
@@ -118,14 +116,7 @@ public class SegmentLoaderHandler extend
                     }
 
                     log.debug("did reread locally corrupt segment " + id + " with size " + s.size());
-                    ByteArrayOutputStream bout = new ByteArrayOutputStream(s.size());
-                    try {
-                        s.writeTo(bout);
-                    } catch (IOException f) {
-                        log.error("can't wrap segment to output stream", f);
-                        throw e;
-                    }
-                    store.writeSegment(s.getSegmentId(), bout.toByteArray(), 0, s.size());
+                    store.persist(s.getSegmentId(), s);
                 }
             } while(true);
             boolean ok = store.setHead(before, builder.getNodeState());
@@ -144,6 +135,7 @@ public class SegmentLoaderHandler extend
                 }
             }
         } finally {
+            store.postSync();
             close();
         }
     }

Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/main/java/org/apache/jackrabbit/oak/plugins/segment/standby/store/StandbyStore.java Wed May  6 07:15:15 2015
@@ -16,11 +16,16 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment.standby.store;
 
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.util.ArrayDeque;
 import java.util.Deque;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import org.apache.jackrabbit.oak.api.Blob;
@@ -70,35 +75,88 @@ public class StandbyStore implements Seg
 
     @Override
     public Segment readSegment(SegmentId sid) {
+        callId++;
         Deque<SegmentId> ids = new ArrayDeque<SegmentId>();
         ids.offer(sid);
         int err = 0;
-        Set<SegmentId> seen = new HashSet<SegmentId>();
+        Set<SegmentId> persisted = new HashSet<SegmentId>();
+
+        Map<SegmentId, Segment> cache = new HashMap<SegmentId, Segment>();
+        long cacheOps = 0;
+
+        long cacheWeight = 0;
+        long maxWeight = 0;
+        long maxKeys = 0;
 
         while (!ids.isEmpty()) {
             SegmentId id = ids.remove();
-            if (!seen.contains(id) && !delegate.containsSegment(id)) {
-                log.debug("trying to read segment " + id);
-                Segment s = loader.readSegment(id.toString());
+            if (!persisted.contains(id) && !delegate.containsSegment(id)) {
+                Segment s;
+                boolean logRefs = true;
+                if (cache.containsKey(id)) {
+                    s = cache.remove(id);
+                    cacheWeight -= s.size();
+                    cacheOps++;
+                    logRefs = false;
+                } else {
+                    log.debug("transferring segment {}", id);
+                    s = loader.readSegment(id.toString());
+                }
+
                 if (s != null) {
-                    log.debug("got segment " + id + " with size " + s.size());
-                    ByteArrayOutputStream bout = new ByteArrayOutputStream(
+                    log.debug("processing segment {} with size {}", id,
                             s.size());
                     if (id.isDataSegmentId()) {
-                        ids.addAll(s.getReferencedIds());
+                        boolean hasPendingRefs = false;
+                        List<SegmentId> refs = s.getReferencedIds();
+                        if (logRefs) {
+                            log.debug("{} -> {}", id, refs);
+                        }
+                        for (SegmentId nr : refs) {
+                            // skip already persisted or self-ref
+                            if (persisted.contains(nr) || id.equals(nr)) {
+                                continue;
+                            }
+                            hasPendingRefs = true;
+                            if (!ids.contains(nr)) {
+                                if (nr.isBulkSegmentId()) {
+                                    // binaries first
+                                    ids.addFirst(nr);
+                                } else {
+                                    // data segments last
+                                    ids.add(nr);
+                                }
+                            }
+                        }
+
+                        if (!hasPendingRefs) {
+                            persisted.add(id);
+                            persist(id, s);
+                        } else {
+                            // persist it later, after the refs are in place
+                            ids.add(id);
+
+                            // TODO there is a chance this might introduce
+                            // a OOME because of the position of the current
+                            // segment in the processing queue. putting it at
+                            // the end of the current queue means it will stay
+                            // in the cache until the pending queue of the
+                            // segment's references is processed.
+                            cache.put(id, s);
+                            cacheWeight += s.size();
+                            cacheOps++;
+
+                            maxWeight = Math.max(maxWeight, cacheWeight);
+                            maxKeys = Math.max(maxKeys, cache.size());
+                        }
+                    } else {
+                        persisted.add(id);
+                        persist(id, s);
                     }
-                    try {
-                        s.writeTo(bout);
-                        writeSegment(id, bout.toByteArray(), 0, s.size());
-                    } catch (IOException e) {
-                        throw new IllegalStateException(
-                                "Unable to write remote segment " + id, e);
-                    }
-                    seen.add(id);
-                    ids.removeAll(seen);
+                    ids.removeAll(persisted);
                     err = 0;
                 } else {
-                    log.error("could NOT read segment " + id);
+                    log.error("could NOT read segment {}", id);
                     if (loader.isClosed() || err == 4) {
                         loader.close();
                         throw new IllegalStateException(
@@ -108,14 +166,44 @@ public class StandbyStore implements Seg
                     ids.addFirst(id);
                 }
             } else {
-                seen.add(id);
+                persisted.add(id);
             }
         }
-
-        log.debug("calling delegate to return segment " + sid);
+        cacheStats.put(callId, "W: " + humanReadableByteCount(maxWeight)
+                + ", Keys: " + maxKeys + ", Ops: " + cacheOps);
         return delegate.readSegment(sid);
     }
 
+    public void persist(SegmentId in, Segment s) {
+        SegmentId id = delegate.getTracker().getSegmentId(
+                in.getMostSignificantBits(), in.getLeastSignificantBits());
+        log.debug("persisting segment {} with size {}", id, s.size());
+        try {
+            ByteArrayOutputStream bout = new ByteArrayOutputStream(s.size());
+            s.writeTo(bout);
+            writeSegment(id, bout.toByteArray(), 0, s.size());
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to write remote segment "
+                    + id, e);
+        }
+    }
+
+    private long callId = 0;
+    private Map<Long, String> cacheStats;
+
+    public void preSync(RemoteSegmentLoader loader) {
+        this.loader = loader;
+        this.cacheStats = new HashMap<Long, String>();
+    }
+
+    public void postSync() {
+        loader = null;
+        if (log.isDebugEnabled() && !cacheStats.isEmpty()) {
+            log.debug("sync cache stats {}", cacheStats);
+        }
+        cacheStats = null;
+    }
+
     @Override
     public void writeSegment(SegmentId id, byte[] bytes, int offset, int length) {
         delegate.writeSegment(id, bytes, offset, length);
@@ -141,10 +229,6 @@ public class StandbyStore implements Seg
         delegate.gc();
     }
 
-    public void setLoader(RemoteSegmentLoader loader) {
-        this.loader = loader;
-    }
-
     public long size() {
         if (delegate instanceof FileStore) {
             try {

Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/test/java/org/apache/jackrabbit/oak/plugins/segment/standby/StandbyTestIT.java Wed May  6 07:15:15 2015
@@ -18,14 +18,12 @@
  */
 package org.apache.jackrabbit.oak.plugins.segment.standby;
 
-import static org.apache.jackrabbit.oak.plugins.segment.SegmentTestUtils.createTmpTargetDir;
 import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.ByteArrayInputStream;
-import java.io.File;
 import java.io.IOException;
 import java.util.Random;
 
@@ -103,11 +101,13 @@ public class StandbyTestIT extends TestB
 
         try {
 
-            for (int i = 0; i < 10; i++) {
+            for (int i = 0; i < 5; i++) {
                 String cp = store.checkpoint(Long.MAX_VALUE);
                 cl.run();
                 assertEquals(primary.getHead(), secondary.getHead());
                 assertTrue(store.release(cp));
+                cl.cleanup();
+                assertTrue(secondary.size() > blobSize);
             }
 
         } finally {

Modified: jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml?rev=1677939&r1=1677938&r2=1677939&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml (original)
+++ jackrabbit/oak/trunk/oak-tarmk-standby/src/test/resources/logback-test.xml Wed May  6 07:15:15 2015
@@ -22,11 +22,18 @@
         </encoder>
     </appender>
 
+    <appender name="file" class="ch.qos.logback.core.FileAppender">
+        <file>target/unit-tests.log</file>
+        <encoder>
+            <pattern>%date{HH:mm:ss.SSS} %-5level %-40([%thread] %F:%L) %msg%n</pattern>
+        </encoder>
+    </appender>
+
     <logger name="org.apache.jackrabbit.oak.plugins.segment.standby" level="ERROR"/>
     <logger name="org.apache.jackrabbit.oak.plugins.blob.datastore" level="ERROR"/>
 
     <root level="INFO">
-        <appender-ref ref="console"/>
+        <appender-ref ref="file"/>
     </root>
 
 </configuration>