You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2018/08/31 12:07:46 UTC
svn commit: r1839746 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment:
Segment.java SegmentBufferWriter.java SegmentDump.java
Author: frm
Date: Fri Aug 31 12:07:46 2018
New Revision: 1839746
URL: http://svn.apache.org/viewvc?rev=1839746&view=rev
Log:
OAK-7720 - Log hex dumps of too big segments
When a SegmentBufferWriter flushes the segment buffer to disk and detects that
the serialized data is too big, it throws an ISE. The information embedded in
the ISE, though, is not enough to debug low-level corruptions in the buffer. In
order to provide more detailed information, SegmentBufferWriter has been
modified to print a dump of the segment buffer and metadata to the log in a
WARN message.
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java (with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1839746&r1=1839745&r2=1839746&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java Fri Aug 31 12:07:46 2018
@@ -35,8 +35,7 @@ import static org.apache.jackrabbit.oak.
import java.io.IOException;
import java.io.OutputStream;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.PrintStream;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Iterator;
@@ -46,7 +45,6 @@ import com.google.common.base.Charsets;
import com.google.common.collect.AbstractIterator;
import org.apache.commons.io.HexDump;
import org.apache.commons.io.output.ByteArrayOutputStream;
-import org.apache.commons.io.output.WriterOutputStream;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.StringUtils;
@@ -547,34 +545,21 @@ public class Segment {
@Override
public String toString() {
- StringWriter string = new StringWriter();
- try (PrintWriter writer = new PrintWriter(string)) {
- writer.format("Segment %s (%d bytes)%n", id, data.size());
- String segmentInfo = getSegmentInfo();
- if (segmentInfo != null) {
- writer.format("Info: %s, Generation: %s%n", segmentInfo, getGcGeneration());
- }
- if (id.isDataSegmentId()) {
- writer.println("--------------------------------------------------------------------------");
- int i = 1;
- for (SegmentId segmentId : segmentReferences) {
- writer.format("reference %02x: %s%n", i++, segmentId);
- }
- for (Entry entry : recordNumbers) {
- int offset = entry.getOffset();
- writer.format("%10s record %08x: %08x @ %08x%n",
- entry.getType(), entry.getRecordNumber(), offset, getAddress(offset));
+ return SegmentDump.dumpSegment(
+ id,
+ data.size(),
+ info,
+ getGcGeneration(),
+ segmentReferences,
+ recordNumbers,
+ stream -> {
+ try {
+ data.hexDump(stream);
+ } catch (IOException e) {
+ e.printStackTrace(new PrintStream(stream));
}
}
- writer.println("--------------------------------------------------------------------------");
- try {
- data.hexDump(new WriterOutputStream(writer, Charsets.UTF_8));
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- writer.println("--------------------------------------------------------------------------");
- }
- return string.toString();
+ );
}
public void writeTo(OutputStream stream) throws IOException {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1839746&r1=1839745&r2=1839746&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Fri Aug 31 12:07:46 2018
@@ -30,6 +30,7 @@ import static java.lang.System.identityH
import static org.apache.jackrabbit.oak.segment.Segment.GC_FULL_GENERATION_OFFSET;
import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET;
import static org.apache.jackrabbit.oak.segment.Segment.HEADER_SIZE;
+import static org.apache.jackrabbit.oak.segment.Segment.MAX_SEGMENT_SIZE;
import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES;
import static org.apache.jackrabbit.oak.segment.Segment.RECORD_SIZE;
import static org.apache.jackrabbit.oak.segment.Segment.SEGMENT_REFERENCE_SIZE;
@@ -38,9 +39,11 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
import java.io.IOException;
+import java.io.PrintStream;
import java.util.Collection;
import java.util.Set;
+import org.apache.commons.io.HexDump;
import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry;
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
import org.jetbrains.annotations.NotNull;
@@ -175,7 +178,7 @@ public class SegmentBufferWriter impleme
* The segment meta data is guaranteed to be the first string record in a segment.
*/
private void newSegment(SegmentStore store) throws IOException {
- buffer = new byte[Segment.MAX_SEGMENT_SIZE];
+ buffer = new byte[MAX_SEGMENT_SIZE];
buffer[0] = '0';
buffer[1] = 'a';
buffer[2] = 'K';
@@ -283,6 +286,24 @@ public class SegmentBufferWriter impleme
dirty = true;
}
+ private String dumpSegmentBuffer() {
+ return SegmentDump.dumpSegment(
+ segment != null ? segment.getSegmentId() : null,
+ length,
+ segment != null ? segment.getSegmentInfo() : null,
+ gcGeneration,
+ segmentReferences,
+ recordNumbers,
+ stream -> {
+ try {
+ HexDump.dump(buffer, 0, stream, 0);
+ } catch (IOException e) {
+ e.printStackTrace(new PrintStream(stream));
+ }
+ }
+ );
+ }
+
/**
* Adds a segment header to the buffer and writes a segment to the segment
* store. This is done automatically (called from prepare) when there is not
@@ -302,6 +323,7 @@ public class SegmentBufferWriter impleme
int totalLength = align(HEADER_SIZE + referencedSegmentIdCount * SEGMENT_REFERENCE_SIZE + recordNumberCount * RECORD_SIZE + length, 16);
if (totalLength > buffer.length) {
+ LOG.warn("Segment buffer corruption detected\n{}", dumpSegmentBuffer());
throw new IllegalStateException(String.format(
"Too much data for a segment %s (referencedSegmentIdCount=%d, recordNumberCount=%d, length=%d, totalLength=%d)",
segment.getSegmentId(), referencedSegmentIdCount, recordNumberCount, length, totalLength));
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java?rev=1839746&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java Fri Aug 31 12:07:46 2018
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+import static org.apache.jackrabbit.oak.segment.Segment.MAX_SEGMENT_SIZE;
+
+import java.io.OutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.function.Consumer;
+
+import com.google.common.base.Charsets;
+import org.apache.commons.io.output.WriterOutputStream;
+import org.apache.jackrabbit.oak.segment.RecordNumbers.Entry;
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+
+class SegmentDump {
+
+ private static int getAddress(int length, int offset) {
+ return length - (MAX_SEGMENT_SIZE - offset);
+ }
+
+ static String dumpSegment(SegmentId id, int length, String segmentInfo, GCGeneration generation, SegmentReferences segmentReferences, RecordNumbers recordNumbers, Consumer<OutputStream> dumper) {
+ StringWriter string = new StringWriter();
+ try (PrintWriter writer = new PrintWriter(string)) {
+ writer.format("Segment %s (%d bytes)%n", id, length);
+ if (segmentInfo != null) {
+ writer.format("Info: %s, Generation: %s%n", segmentInfo, generation);
+ }
+ if (id != null && id.isDataSegmentId()) {
+ writer.println("--------------------------------------------------------------------------");
+ int i = 1;
+ for (SegmentId segmentId : segmentReferences) {
+ writer.format("reference %02x: %s%n", i++, segmentId);
+ }
+ for (Entry entry : recordNumbers) {
+ int offset = entry.getOffset();
+ writer.format("%10s record %08x: %08x @ %08x%n",
+ entry.getType(), entry.getRecordNumber(), offset, getAddress(length, offset));
+ }
+ }
+ writer.println("--------------------------------------------------------------------------");
+ dumper.accept(new WriterOutputStream(writer, Charsets.UTF_8));
+ writer.println("--------------------------------------------------------------------------");
+ }
+ return string.toString();
+ }
+
+}
+
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentDump.java
------------------------------------------------------------------------------
svn:eol-style = native