You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2026/08/20 18:29:41 UTC
(tinkerpop) 02/09: Encode TinkerStorageGraph elements as dictionary-referenced components
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit dc72ba579a1245ee774a81924e9a9e792d7aa79a
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 14:31:18 2026 +0000
Encode TinkerStorageGraph elements as dictionary-referenced components
Replace whole-DetachedVertex/DetachedEdge GraphBinary serialization with a
component codec: element ids and property values go through GraphBinary's
scalar serializers (a one-byte DataType tag + raw value), while labels,
property keys, and meta-property keys are dictionary-encoded to small integer
refs. New keys are emitted as OP_DICT_APPEND records within the same frame,
before their first use. Vertex-property ids are auto-generated and no longer
persisted (regenerated on load); element and edge ids are preserved.
Compaction preserves dictionary numbering and writes the whole dictionary as a
self-contained snapshot header, and dict-append decode is idempotent, so a log
surviving the compaction crash window still resolves its refs. This removes the
old whole-object codec (no dead code) and drops per-element key/label
repetition and the per-property envelope, shrinking the on-disk footprint.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 371 ++++++++++++++++++---
.../structure/storage/GraphBinaryStorageTest.java | 14 +-
.../storage/StorageCrashConsistencyTest.java | 15 +-
3 files changed, 334 insertions(+), 66 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index a5f3fd1c5c..1266dee705 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -19,34 +19,51 @@
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.binary.DataType;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter;
+import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializer;
import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* The GraphBinary {@link TinkerStorage} codec on top of {@link AbstractLogStorage}. The base owns the durable
* log-structured machinery (file layout, {@code VERSION} marker, CRC framing, replay fold, {@link SyncMode}
- * durability, crash-safe and threshold compaction); this class supplies only how an element is encoded and decoded,
- * using the GraphBinary serializers ({@link GraphBinaryWriter}/{@link GraphBinaryReader}).
+ * durability, crash-safe and threshold compaction); this class supplies only how an element is encoded and decoded.
* <p/>
- * Known limitation (write amplification): a commit records each changed element in full — a single property change on
- * a large element rewrites the whole element to the log. Elements are typically small and automatic compaction bounds
- * the resulting log growth, so this is accepted rather than mitigated with per-property deltas, which would complicate
- * the {@link TinkerStorageMutation} contract and the replay fold. The snapshot, by contrast, is streamed one element
- * at a time so compaction never holds a second full copy of the graph in heap.
+ * Rather than serialize a whole {@code DetachedVertex}/{@code DetachedEdge} (which repeats every property key and
+ * label as a full string on every element and wraps each property in a {@code VertexProperty} envelope), this codec
+ * writes element <em>components</em> directly: element ids and property values go through GraphBinary's scalar
+ * serializers (a one-byte {@link DataType} tag plus the raw value), while labels, property keys, and meta-property
+ * keys are dictionary-encoded to small integer refs. The dictionary is a single dense namespace built by first
+ * appearance; new entries are emitted as {@code OP_DICT_APPEND} records inside the same frame, before the entries
+ * that reference them, so a torn trailing frame drops a ref and its user atomically.
+ * <p/>
+ * Vertex-property ids are auto-generated and, by default, not persisted (they are regenerated on load); element and
+ * edge ids are always preserved. The snapshot streams one element per frame, so compaction never holds a second full
+ * copy of the graph in heap.
*/
public final class GraphBinaryStorage extends AbstractLogStorage {
@@ -54,71 +71,230 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
private static final byte OP_DEL_VERTEX = 2;
private static final byte OP_PUT_EDGE = 3;
private static final byte OP_DEL_EDGE = 4;
+ private static final byte OP_DICT_APPEND = 5;
- private final GraphBinaryWriter writer = new GraphBinaryWriter(TypeSerializerRegistry.INSTANCE);
- private final GraphBinaryReader reader = new GraphBinaryReader(TypeSerializerRegistry.INSTANCE);
+ private final TypeSerializerRegistry registry = TypeSerializerRegistry.INSTANCE;
+ private final GraphBinaryWriter writer = new GraphBinaryWriter(registry);
+ private final GraphBinaryReader reader = new GraphBinaryReader(registry);
/**
- * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
- * serialized element (put) or the serialized id (delete).
+ * Shared string dictionary for labels, property keys, and meta-property keys. Dense ids assigned by first
+ * appearance; grows monotonically within a write session and is rebuilt fresh on compaction and on replay.
*/
+ private final Map<String, Integer> keyToId = new HashMap<>();
+ private final List<String> idToKey = new ArrayList<>();
+
+ @Override
+ protected void beginReplay() {
+ keyToId.clear();
+ idToKey.clear();
+ }
+
+ // --------------------------------------------------------------------------------------------- encode
+
@Override
protected byte[] encodeCommit(final long txVersion,
final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(txVersion);
- buffer.writeInt(changedVertices.size() + changedEdges.size());
+ final ByteBufferBuffer buf = new ByteBufferBuffer();
+ // register the strings introduced by this commit (deletes carry only an id, no strings)
+ final List<String> appends = new ArrayList<>();
+ for (final TinkerStorageMutation<TinkerVertex> m : changedVertices)
+ if (!m.isDeleted()) registerVertexStrings(m.element(), appends);
+ for (final TinkerStorageMutation<TinkerEdge> m : changedEdges)
+ if (!m.isDeleted()) registerEdgeStrings(m.element(), appends);
+
+ writeVarInt(buf, appends.size() + changedVertices.size() + changedEdges.size());
+ // dictionary appends first, so every ref below resolves during the fold
+ for (final String s : appends) {
+ buf.writeByte(OP_DICT_APPEND);
+ writeVarInt(buf, keyToId.get(s));
+ writeString(buf, s);
+ }
for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_VERTEX);
- writer.write(m.id(), buffer);
+ buf.writeByte(OP_DEL_VERTEX);
+ writeScalar(buf, m.id());
} else {
- buffer.writeByte(OP_PUT_VERTEX);
- // detach to a stable form independent of the transactional element
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ buf.writeByte(OP_PUT_VERTEX);
+ writeVertexRecord(buf, m.element());
}
}
for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_EDGE);
- writer.write(m.id(), buffer);
+ buf.writeByte(OP_DEL_EDGE);
+ writeScalar(buf, m.id());
} else {
- buffer.writeByte(OP_PUT_EDGE);
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ buf.writeByte(OP_PUT_EDGE);
+ writeEdgeRecord(buf, m.element());
+ }
+ }
+ return buf.toWrittenArray();
+ }
+
+ @Override
+ protected void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
+ // Preserve the existing dictionary numbering rather than renumbering: the compaction crash window can leave
+ // the new snapshot in place with the old log not yet truncated, and that log's refs use the current
+ // numbering. Register any not-yet-seen live strings (this only extends the dictionary, never renumbers), then
+ // emit the whole dictionary as a self-contained header frame so a snapshot-only replay resolves every ref.
+ final List<String> ignored = new ArrayList<>();
+ Iterator<Vertex> vertices = graph.vertices();
+ while (vertices.hasNext())
+ registerVertexStrings(vertices.next(), ignored);
+ Iterator<Edge> edges = graph.edges();
+ while (edges.hasNext())
+ registerEdgeStrings(edges.next(), ignored);
+
+ final ByteBufferBuffer dictBuf = new ByteBufferBuffer();
+ writeVarInt(dictBuf, idToKey.size());
+ for (int id = 0; id < idToKey.size(); id++) {
+ dictBuf.writeByte(OP_DICT_APPEND);
+ writeVarInt(dictBuf, id);
+ writeString(dictBuf, idToKey.get(id));
+ }
+ writeFrame(out, dictBuf.toWrittenArray());
+
+ // one element per frame; all keys are already in the dictionary header, so no per-frame appends
+ vertices = graph.vertices();
+ while (vertices.hasNext())
+ writeElementFrame(out, OP_PUT_VERTEX, vertices.next());
+ edges = graph.edges();
+ while (edges.hasNext())
+ writeElementFrame(out, OP_PUT_EDGE, edges.next());
+ }
+
+ /**
+ * Write one element as its own single-entry put frame. One element is held in memory at a time, so a large graph
+ * is never buffered whole.
+ */
+ private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
+ final ByteBufferBuffer buf = new ByteBufferBuffer();
+ writeVarInt(buf, 1);
+ buf.writeByte(op);
+ if (op == OP_PUT_VERTEX) writeVertexRecord(buf, (Vertex) element);
+ else writeEdgeRecord(buf, (Edge) element);
+ writeFrame(out, buf.toWrittenArray());
+ }
+
+ private void registerVertexStrings(final Vertex v, final List<String> appends) {
+ for (final String label : v.labels())
+ register(label, appends);
+ final Iterator<VertexProperty<Object>> vps = v.properties();
+ while (vps.hasNext()) {
+ final VertexProperty<Object> vp = vps.next();
+ register(vp.key(), appends);
+ final Iterator<Property<Object>> metas = vp.properties();
+ while (metas.hasNext())
+ register(metas.next().key(), appends);
+ }
+ }
+
+ private void registerEdgeStrings(final Edge e, final List<String> appends) {
+ register(e.label(), appends);
+ final Iterator<Property<Object>> props = e.properties();
+ while (props.hasNext())
+ register(props.next().key(), appends);
+ }
+
+ private void register(final String s, final List<String> appends) {
+ if (!keyToId.containsKey(s)) {
+ final int id = idToKey.size();
+ keyToId.put(s, id);
+ idToKey.add(s);
+ appends.add(s);
+ }
+ }
+
+ private void writeVertexRecord(final ByteBufferBuffer buf, final Vertex v) throws IOException {
+ writeScalar(buf, v.id());
+ final Set<String> labels = v.labels();
+ writeVarInt(buf, labels.size());
+ for (final String label : labels)
+ writeVarInt(buf, keyToId.get(label));
+
+ // group vertex properties by key so multi-properties (list/set) round-trip
+ final Map<String, List<VertexProperty<Object>>> groups = new LinkedHashMap<>();
+ final Iterator<VertexProperty<Object>> vps = v.properties();
+ while (vps.hasNext()) {
+ final VertexProperty<Object> vp = vps.next();
+ groups.computeIfAbsent(vp.key(), k -> new ArrayList<>()).add(vp);
+ }
+ writeVarInt(buf, groups.size());
+ for (final Map.Entry<String, List<VertexProperty<Object>>> group : groups.entrySet()) {
+ writeVarInt(buf, keyToId.get(group.getKey()));
+ final List<VertexProperty<Object>> values = group.getValue();
+ writeVarInt(buf, values.size());
+ for (final VertexProperty<Object> vp : values) {
+ writeScalar(buf, vp.value());
+ final List<Property<Object>> metas = new ArrayList<>();
+ vp.properties().forEachRemaining(metas::add);
+ writeVarInt(buf, metas.size());
+ for (final Property<Object> meta : metas) {
+ writeVarInt(buf, keyToId.get(meta.key()));
+ writeScalar(buf, meta.value());
+ }
}
}
- return buffer.toWrittenArray();
}
+ private void writeEdgeRecord(final ByteBufferBuffer buf, final Edge e) throws IOException {
+ writeScalar(buf, e.id());
+ writeVarInt(buf, keyToId.get(e.label()));
+ writeScalar(buf, e.outVertex().id());
+ writeScalar(buf, e.inVertex().id());
+ final List<Property<Object>> props = new ArrayList<>();
+ e.properties().forEachRemaining(props::add);
+ writeVarInt(buf, props.size());
+ for (final Property<Object> p : props) {
+ writeVarInt(buf, keyToId.get(p.key()));
+ writeScalar(buf, p.value());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------- decode
+
@Override
protected void decodeFrame(final byte[] record,
final Map<Object, DetachedVertex> vertices,
final Map<Object, DetachedEdge> edges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
- buffer.readLong(); // txVersion, retained for diagnostics/future use
- final int entryCount = buffer.readInt();
+ final ByteBufferBuffer buf = new ByteBufferBuffer(record);
+ final int entryCount = readVarInt(buf);
for (int i = 0; i < entryCount; i++) {
- final byte op = buffer.readByte();
+ final byte op = buf.readByte();
switch (op) {
+ case OP_DICT_APPEND: {
+ final int id = readVarInt(buf);
+ final String s = readString(buf);
+ // idempotent: a snapshot header defines the whole dictionary, and a log surviving the compaction
+ // crash window may re-append entries the snapshot already established. Re-appending an existing
+ // id with the same string is a no-op; a mismatch or a gap is corruption.
+ if (id < idToKey.size()) {
+ if (!idToKey.get(id).equals(s))
+ throw new IOException(String.format("Corrupt storage: dictionary id %d redefined ('%s' vs '%s')", id, idToKey.get(id), s));
+ } else if (id == idToKey.size()) {
+ idToKey.add(s);
+ } else {
+ throw new IOException(String.format("Corrupt storage: dictionary append gap (got %d, expected <= %d)", id, idToKey.size()));
+ }
+ break;
+ }
case OP_PUT_VERTEX: {
- final Vertex v = reader.read(buffer);
- vertices.put(v.id(), (DetachedVertex) v);
+ final DetachedVertex v = readVertexRecord(buf);
+ vertices.put(v.id(), v);
break;
}
case OP_DEL_VERTEX: {
- final Object id = reader.read(buffer);
- vertices.remove(id);
+ vertices.remove(readScalar(buf));
break;
}
case OP_PUT_EDGE: {
- final Edge e = reader.read(buffer);
- edges.put(e.id(), (DetachedEdge) e);
+ final DetachedEdge e = readEdgeRecord(buf);
+ edges.put(e.id(), e);
break;
}
case OP_DEL_EDGE: {
- final Object id = reader.read(buffer);
- edges.remove(id);
+ edges.remove(readScalar(buf));
break;
}
default:
@@ -127,29 +303,114 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
}
}
+ private DetachedVertex readVertexRecord(final ByteBufferBuffer buf) throws IOException {
+ final Object id = readScalar(buf);
+ final DetachedVertex.Builder b = DetachedVertex.build().setId(id);
+ final int labelCount = readVarInt(buf);
+ if (labelCount == 1) {
+ b.setLabel(idToKey.get(readVarInt(buf)));
+ } else if (labelCount > 1) {
+ final Set<String> labels = new LinkedHashSet<>();
+ for (int i = 0; i < labelCount; i++)
+ labels.add(idToKey.get(readVarInt(buf)));
+ b.setLabels(labels);
+ }
+ final int keyGroupCount = readVarInt(buf);
+ for (int g = 0; g < keyGroupCount; g++) {
+ final String key = idToKey.get(readVarInt(buf));
+ final int valueCount = readVarInt(buf);
+ for (int j = 0; j < valueCount; j++) {
+ final Object value = readScalar(buf);
+ final DetachedVertexProperty.Builder vpb = DetachedVertexProperty.build().setLabel(key).setValue(value);
+ final int metaCount = readVarInt(buf);
+ for (int m = 0; m < metaCount; m++) {
+ final String metaKey = idToKey.get(readVarInt(buf));
+ final Object metaValue = readScalar(buf);
+ vpb.addProperty(new DetachedProperty<>(metaKey, metaValue));
+ }
+ b.addProperty(vpb.create());
+ }
+ }
+ return b.create();
+ }
+
+ private DetachedEdge readEdgeRecord(final ByteBufferBuffer buf) throws IOException {
+ final Object id = readScalar(buf);
+ final String label = idToKey.get(readVarInt(buf));
+ final Object outVId = readScalar(buf);
+ final Object inVId = readScalar(buf);
+ final DetachedEdge.Builder b = DetachedEdge.build().setId(id).setLabel(label)
+ .setOutV(DetachedVertex.build().setId(outVId).create())
+ .setInV(DetachedVertex.build().setId(inVId).create());
+ final int propCount = readVarInt(buf);
+ for (int i = 0; i < propCount; i++) {
+ final String key = idToKey.get(readVarInt(buf));
+ final Object value = readScalar(buf);
+ b.addProperty(new DetachedProperty<>(key, value));
+ }
+ return b.create();
+ }
+
+ // --------------------------------------------------------------------------------------------- primitives
+
/**
- * Stream the current committed state as one framed put record per vertex and per edge, so peak memory is bounded
- * to a single element rather than materializing the whole graph as one byte array.
+ * Write a value as a one-byte {@link DataType} tag followed by the raw value (no value-flag byte). A {@code null}
+ * is a single {@link DataType#UNSPECIFIED_NULL} tag.
*/
- @Override
- protected void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
- final Iterator<Vertex> vertexIterator = graph.vertices();
- while (vertexIterator.hasNext())
- writeElementFrame(out, OP_PUT_VERTEX, vertexIterator.next());
- final Iterator<Edge> edgeIterator = graph.edges();
- while (edgeIterator.hasNext())
- writeElementFrame(out, OP_PUT_EDGE, edgeIterator.next());
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void writeScalar(final ByteBufferBuffer buf, final Object value) throws IOException {
+ if (value == null) {
+ buf.writeByte(DataType.UNSPECIFIED_NULL.getCodeByte());
+ return;
+ }
+ final TypeSerializer serializer = registry.getSerializer(value.getClass());
+ buf.writeByte(serializer.getDataType().getCodeByte());
+ serializer.writeValue(value, buf, writer, false);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private Object readScalar(final ByteBufferBuffer buf) throws IOException {
+ final DataType dataType = DataType.get(Byte.toUnsignedInt(buf.readByte()));
+ if (dataType == DataType.UNSPECIFIED_NULL)
+ return null;
+ final TypeSerializer serializer = registry.getSerializer(dataType);
+ return serializer.readValue(buf, reader, false);
+ }
+
+ private static void writeString(final ByteBufferBuffer buf, final String s) {
+ final byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ writeVarInt(buf, bytes.length);
+ buf.writeBytes(bytes);
+ }
+
+ private static String readString(final ByteBufferBuffer buf) {
+ final byte[] bytes = new byte[readVarInt(buf)];
+ buf.readBytes(bytes);
+ return new String(bytes, StandardCharsets.UTF_8);
}
/**
- * Encode a single element as a one-entry put record and write it as a framed record to {@code out}.
+ * Unsigned LEB128 varint. Counts and dictionary refs are small and non-negative, so they cost one byte in the
+ * common case.
*/
- private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(0L); // snapshot records have no single tx version
- buffer.writeInt(1);
- buffer.writeByte(op);
- writer.write(DetachedFactory.detach(element, true), buffer);
- writeFrame(out, buffer.toWrittenArray());
+ private static void writeVarInt(final ByteBufferBuffer buf, final int value) {
+ int v = value;
+ while ((v & ~0x7F) != 0) {
+ buf.writeByte((v & 0x7F) | 0x80);
+ v >>>= 7;
+ }
+ buf.writeByte(v & 0x7F);
+ }
+
+ private static int readVarInt(final ByteBufferBuffer buf) {
+ int result = 0;
+ int shift = 0;
+ byte b;
+ do {
+ b = buf.readByte();
+ result |= (b & 0x7F) << shift;
+ shift += 7;
+ } while ((b & 0x80) != 0);
+ return result;
}
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index c113511176..cbde701a22 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -127,10 +127,10 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.tx().commit();
graph.compact();
- // the snapshot must be written as one framed record per element (3 vertices + 2 edges = 5), rather than a
- // single whole-graph frame, so compaction never buffers the entire graph in one array
+ // the snapshot must be streamed: a dictionary header frame plus one framed record per element (3 vertices +
+ // 2 edges = 5), rather than a single whole-graph frame, so compaction never buffers the entire graph at once
final File snapshotFile = new File(location, GraphBinaryStorage.SNAPSHOT_FILE);
- assertEquals(5, countFrames(snapshotFile));
+ assertEquals(1 + 5, countFrames(snapshotFile));
graph.close();
// and the streamed snapshot must reopen to exactly the same graph
@@ -145,9 +145,9 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
@Test
public void shouldStreamSnapshotFrameByFrameAtScale() {
// Bounded-memory proxy for the streaming snapshot path: rather than measure heap (flaky, JVM-dependent), assert
- // the observable streaming property holds at scale — a large graph is written as exactly one frame per element,
- // never one whole-graph frame — and round-trips intact. This is the property that keeps compaction from
- // materializing a second full copy of the graph in memory; it is not a hard OOM assertion.
+ // the observable streaming property holds at scale — a large graph is written as a dictionary header frame plus
+ // one frame per element, never one whole-graph frame — and round-trips intact. This is the property that keeps
+ // compaction from materializing a second full copy of the graph in memory; it is not a hard OOM assertion.
final int vertexCount = 500;
final int edgeCount = 499;
final TinkerStorageGraph graph = open();
@@ -161,7 +161,7 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.compact();
try {
- assertEquals(vertexCount + edgeCount, countFrames(new File(location, GraphBinaryStorage.SNAPSHOT_FILE)));
+ assertEquals(1 + vertexCount + edgeCount, countFrames(new File(location, GraphBinaryStorage.SNAPSHOT_FILE)));
} catch (Exception ex) {
throw new RuntimeException(ex);
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
index 614cfdb0ba..8f3c475f13 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
@@ -123,10 +123,17 @@ public class StorageCrashConsistencyTest {
@Test
public void shouldRecoverDurableCommitWithNoSnapshot() throws Exception {
- // WAL guarantee: a commit whose frame was durably written to the log, with no compaction having run, must
- // recover on reopen even though no snapshot exists.
- captureBuildingBlocks();
- Files.write(logFile.toPath(), logV2);
+ // WAL guarantee: a commit durably written to the log with no compaction (no snapshot) recovers on reopen. On
+ // a fresh store the dictionary starts empty, so the commit frame is self-contained (it carries its own
+ // dictionary appends), which is exactly the real "log only, never compacted" case.
+ final TinkerStorageGraph g = open();
+ g.addVertex(T.id, 2, "value", 2);
+ g.tx().commit();
+ g.tx().close();
+ final byte[] selfContainedLog = Files.readAllBytes(logFile.toPath());
+ g.close(); // compaction on close would fold the log away; the raw log was captured above
+ resetFiles();
+ Files.write(logFile.toPath(), selfContainedLog);
assertReopensTo(2);
}