You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2026/09/04 13:49:05 UTC
(tinkerpop) branch tinkergraph-storage updated (819bd8c17c -> 9ebb314541)
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a change to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
discard 819bd8c17c Document TinkerStorageGraph persistence and fix stale class doc
discard 73dbe40cb2 Document TinkerStorageGraph single-writer stance; enforce meta-property conflict
discard 7788cb2bd1 Reject corrupt TinkerStorage frames with a clear error
discard 379deede92 Test TinkerStorage durability, lifecycle, and dictionary crash window
discard f32a598ea3 Broaden TinkerStorage TCK: value types, ids, varints, unicode round-trips
discard a7273bc473 Document TinkerStorageGraph compact encoding and vertex-property id option
discard 7832c81a9c Add opt-in vertex-property id persistence to TinkerStorageGraph storage
discard dc72ba579a Encode TinkerStorageGraph elements as dictionary-referenced components
discard ce004eec0b Extract AbstractLogStorage base from GraphBinaryStorage
discard 1b16518103 Document TinkerStorageGraph durability, compaction, and locking
discard 2273b7dc5e Record TinkerStorageGraph format version once per store
discard 9d23887aed Add crash-consistency and streaming-scale tests for TinkerStorageGraph storage
discard cf97e28a1a Add integrity checks to TinkerStorageGraph storage format
discard d9c62b3511 Stream TinkerStorageGraph snapshots one element at a time
discard 2014c53edf Auto-compact TinkerStorageGraph log to bound its growth
discard f908e0c2b5 Lock TinkerStorageGraph storage directory to a single writer
discard af88e7d948 Serialize concurrent commit writes to TinkerStorageGraph storage
discard 0b2891051a Make TinkerStorageGraph commits fsync-durable with a configurable sync mode
discard 2ccf1c9c06 Move TinkerStorageGraph storage changelog entries to 4.0.0 section
discard c082867975 Add pluggable storage layer to TinkerStorageGraph
add 54550d282f Expand Times Step reference section with example and fix javadoc link text
add 519a7be157 Fix graph.computer() typo in GraphComputer Graph Filter docs
add 7a836a6358 Expand the Filter Step reference documentation
add 574335b33f Expand Map Step reference documentation with examples and contract
add 0e9f184139 Expand To Step reference docs with argument forms and a live example
add a41ce94333 Expand Until Step reference docs with an example and overload note
add 6a2db23096 Fix stale class name and Gryo comment in io() step reference docs
add 1534a850bf Fix GraphSON Java writer snippets to compile against the real API
add 6be474f65b Merge branch 'pr-3627' into 3.7-dev
add 3eb66ae5d9 Fix and clarify the VertexProgram listing in the Program Step docs
add 1c4bbda0f5 Merge branch 'pr-3628' into 3.7-dev
add c5fdf472ce Merge branch '3.7-dev' into 3.8-dev
add 0958b07d9e Merge branch '3.8-dev'
add 751e6f86be Link merged commits back to their beads
add 5818ca9b7d Filter the commit window on committer date
add 78f03e480a Merge branch '3.7-dev' into 3.8-dev
add e9e7f0efcb Merge branch '3.8-dev'
add 6ed2e0c791 Require asking why an alternative was rejected
add ab6e2f0112 Merge branch '3.7-dev' into 3.8-dev
add 8ce8bad3d5 Merge branch '3.8-dev'
add 367bca2ba2 Note that an epic root needs --force to close
add 4d74b360ee Merge branch '3.7-dev' into 3.8-dev
add fd0e3b5be0 Merge branch '3.8-dev'
add ae1b2a88fc Strengthen nested strategy class configuration
add 5cc351e9e7 Merge branch '3.7-dev' into 3.8-dev
add 223c147f9d Merge branch '3.8-dev'
add 5dda51e73e Backport TraversalStrategies.GlobalCache.GLOBAL_REGISTRY to 3.7
add d1a9100eb8 Allow operators to deny traversal strategies globally
add cff2a1339b Restrict GraphBinary traversal strategy deserialization
add fa5fbe43f1 Merge branch '3.7-dev' into 3.8-dev
add 18681892a9 Merge branch '3.8-dev'
add 51a7f29c78 CTR: Verify io() class names before loading and invoking them
add 65f38abf53 Merge branch '3.7-dev' into 3.8-dev
add b9afe89d0e Merge branch '3.8-dev'
add 9ad6232115 Add TinkerPop Sample Data documentation book
add e678e22c4f Merge branch 'samplebook' into 3.7-dev
add a080dfa6d5 Merge branch '3.7-dev' into 3.8-dev
add 1d8a5c825f Merge branch '3.8-dev'
add 1f6a462ac1 Add Air Routes to the Sample Data docs book
add 1a5391215a Merge branch '3.8-dev'
add 193e678506 Add The Zoo to the Sample Data docs book
add 696d8636c4 Restrict GraphSON2/3 traversal strategy deserialization
add 71d8528449 Merge branch '3.8-dev'
add 1f0ded919d CTR: Guard against non-terminating loops in TinkerGraph GraphSON deserialization
add f33bbcb13b Merge branch '3.7-dev' into 3.8-dev
add a0c04d2139 Merge branch '3.8-dev'
add d557266e40 Restrict Class value deserialization in GraphBinary, GraphSON, and Gryo
add 63d94c880d Merge branch '3.7-dev' into 3.8-dev
add 2f3ad0188a Merge branch '3.8-dev'
add ae53653365 Add Remote Restrictions in Hadoop/Spark OLAP
add 82850f7783 Merge remote-tracking branch 'origin/3.7-dev' into 3.8-dev
add 405d53bb34 Merge remote-tracking branch 'origin/3.8-dev'
add 739d5c4a6f Restrict GraphSON 1.0 embedded-type deserialization (#3586)
add 35d8f49b6b Merge branch '3.7-dev' into 3.8-dev
add c2ab68a957 Merge branch '3.8-dev'
add e340913794 Require registered strategies in JavaTranslator CTR
add b644ac0c4c Merge branch '3.7-dev' into 3.8-dev
add 36597457a2 Merge branch '3.8-dev'
add baf2bd2fb2 CTR Bump Netty to 4.1.137.final
add 8e861e5f5b Merge branch '3.7-dev' into 3.8-dev
add 90abe4fe02 Merge branch '3.8-dev'
add ed61d5787e Update threat model CTR
add eae093bbfd Isolate authenticated users in authorization handlers CTR
add 2a30be2ba3 Isolate OLAP io() configuration per request
add c90ac5df94 Ensure buffer release in HTTP handlers CTR
add bc9936c0de Merge branch '3.7-dev' into 3.8-dev
add 0e743cbc82 Fixed P.typeOf()
add 2ed613e544 Merge branch '3.8-dev'
add d429ec0bd8 Cap and expire requests deferred during SASL authentication
add 99c9153c84 Merge branch '3.7-dev' into 3.8-dev
add 8bdc4ef7d6 Merge branch '3.8-dev'
add 76b06f71ba Add toy-graph dataset captions to rendered Gremlin examples CTR
add 2718dbbbf5 Merge remote-tracking branch 'origin/3.7-dev' into 3.8-dev
add 70cd4293c5 Merge branch '3.8-dev'
add a97e7d8866 CTR update Apache Ivy version in NOTICE files
add 9ef293de9d Merge remote-tracking branch 'origin/3.7-dev' into 3.8-dev
add 9199afe4e3 Merge remote-tracking branch 'origin/3.8-dev'
add 86a495f591 TinkerPop 3.7.7 release
add 1a99f5064e Merge branch '3.7-dev' into 3.8-dev
add 952d8d8e4c TinkerPop 3.8.2 release
add b9c8b1edf5 Merge branch '3.8-dev'
new e08c6f823a Add pluggable storage layer to TinkerStorageGraph
new 93702c87a8 Move TinkerStorageGraph storage changelog entries to 4.0.0 section
new 2a258ffba8 Make TinkerStorageGraph commits fsync-durable with a configurable sync mode
new 963217cf1b Serialize concurrent commit writes to TinkerStorageGraph storage
new d09a3a5324 Lock TinkerStorageGraph storage directory to a single writer
new 78474a129d Auto-compact TinkerStorageGraph log to bound its growth
new 6840d08d2c Stream TinkerStorageGraph snapshots one element at a time
new 5d56976223 Add integrity checks to TinkerStorageGraph storage format
new 3f6b48c69b Add crash-consistency and streaming-scale tests for TinkerStorageGraph storage
new 666a098a11 Record TinkerStorageGraph format version once per store
new adcd29a6af Document TinkerStorageGraph durability, compaction, and locking
new 5f331196c9 Extract AbstractLogStorage base from GraphBinaryStorage
new 498bdb1c40 Encode TinkerStorageGraph elements as dictionary-referenced components
new 41423dedbe Add opt-in vertex-property id persistence to TinkerStorageGraph storage
new 571488d52e Document TinkerStorageGraph compact encoding and vertex-property id option
new bc79b74142 Broaden TinkerStorage TCK: value types, ids, varints, unicode round-trips
new a90ef636c3 Test TinkerStorage durability, lifecycle, and dictionary crash window
new 72a09a4db3 Reject corrupt TinkerStorage frames with a clear error
new eb33eb3153 Document TinkerStorageGraph single-writer stance; enforce meta-property conflict
new 9ebb314541 Document TinkerStorageGraph persistence and fix stale class doc
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (819bd8c17c)
\
N -- N -- N refs/heads/tinkergraph-storage (9ebb314541)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 20 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.beads/PRIME.md | 43 +-
.beads/issues.jsonl | 11 +
.gitignore | 3 +
CHANGELOG.asciidoc | 83 ++-
THREAT_MODEL.md | 25 +-
bin/beads-agent-hook.sh | 4 +-
bin/beads-commits.py | 349 ++++++++++++
bin/beads-report.py | 20 +-
bin/validate-distribution.sh | 4 +-
docs/sass/tabs.scss | 53 +-
docs/src/data/air-routes.asciidoc | 145 +++++
docs/src/data/grateful-dead.asciidoc | 124 +++++
docs/src/data/index.asciidoc | 63 +++
docs/src/data/modern.asciidoc | 106 ++++
docs/src/data/the-crew.asciidoc | 122 +++++
docs/src/data/the-zoo.asciidoc | 134 +++++
docs/src/dev/developer/for-committers.asciidoc | 84 ++-
docs/src/dev/developer/release.asciidoc | 3 +
docs/src/dev/io/graphml.asciidoc | 3 +-
docs/src/dev/io/graphson.asciidoc | 40 +-
docs/src/index.asciidoc | 3 +
docs/src/recipes/centrality.asciidoc | 4 +-
docs/src/reference/gremlin-applications.asciidoc | 1 +
.../implementations-hadoop-start.asciidoc | 31 ++
docs/src/reference/intro.asciidoc | 5 +-
docs/src/reference/the-graph.asciidoc | 2 +
docs/src/reference/the-graphcomputer.asciidoc | 2 +-
docs/src/reference/the-traversal.asciidoc | 166 +++++-
docs/src/tutorials/getting-started/index.asciidoc | 3 +
.../tutorials/the-gremlin-console/index.asciidoc | 12 +-
docs/src/upgrade/release-3.7.x.asciidoc | 184 ++++++-
docs/src/upgrade/release-3.8.2.asciidoc | 22 +-
docs/stylesheets/tinkerpop.css | 6 +-
.../apache/tinkerpop/tinkeradoc/GraphCatalog.java | 96 ++++
.../tinkerpop/tinkeradoc/GremlinTreeprocessor.java | 51 +-
.../tinkerpop/tinkeradoc/HtmlTabRenderer.java | 16 +-
.../tinkerpop/tinkeradoc/MarkdownTabRenderer.java | 15 +
.../apache/tinkerpop/tinkeradoc/NeutralTab.java | 6 +-
.../tinkerpop/tinkeradoc/TabbedHtmlBuilder.java | 47 +-
.../tinkeradoc/DualBackendIntegrationTest.java | 2 +-
.../tinkerpop/tinkeradoc/GraphCatalogTest.java | 65 +++
.../tinkeradoc/GremlinTreeprocessorTest.java | 94 +++-
.../tinkerpop/tinkeradoc/HtmlTabRendererTest.java | 4 +-
.../tinkerpop/tinkeradoc/IntegrationTest.java | 2 +-
.../tinkeradoc/MarkdownTabRendererTest.java | 18 +-
.../tinkerpop/tinkeradoc/NeutralTabCodecTest.java | 2 +-
.../tinkeradoc/TabbedHtmlBuilderTest.java | 33 +-
gremlin-console/src/main/static/NOTICE | 6 +-
.../traversal/step/map/VertexProgramStep.java | 14 +-
.../strategy/decoration/VertexProgramStrategy.java | 13 +-
.../process/traversal/TraversalStrategies.java | 62 ++-
.../process/traversal/step/sideEffect/IoStep.java | 86 ++-
.../decoration/HaltedTraverserStrategy.java | 26 +-
.../finalization/MatchAlgorithmStrategy.java | 9 +-
.../structure/io/graphson/GraphSONMapper.java | 235 +++++++-
.../io/graphson/TraversalSerializersV2.java | 12 +-
.../io/graphson/TraversalSerializersV3.java | 12 +-
.../structure/io/util/IoRegistryHelper.java | 20 +-
.../com/example/gadget/GraphSONTestGadgets.java | 127 +++++
.../gremlin/process/TraversalStrategiesTest.java | 221 ++++++++
.../traversal/step/map/VertexProgramStepTest.java | 178 ++++++
.../decoration/VertexProgramStrategyTest.java | 124 +++++
.../traversal/step/sideEffect/IoStepTest.java | 202 +++++++
.../decoration/HaltedTraverserStrategyTest.java | 58 ++
.../finalization/MatchAlgorithmStrategyTest.java | 83 +++
...ensTest.java => GraphSONMapperBuilderTest.java} | 22 +-
.../GraphSONMapperPartialEmbeddedTypeTest.java | 595 +++++++++++++++++++++
.../io/graphson/PGraphSONSerializerTest.java | 72 +++
...versalStrategyProxyJacksonDeserializerTest.java | 127 +++++
.../structure/io/util/IoRegistryHelperTest.java | 146 +++++
gremlin-driver/src/main/static/NOTICE | 2 +-
gremlin-server/pom.xml | 2 +-
.../tinkerpop/gremlin/server/authz/Authorizer.java | 5 +
.../handler/HttpBasicAuthorizationHandler.java | 10 +-
gremlin-server/src/main/static/NOTICE | 6 +-
.../handler/HttpBasicAuthorizationHandlerTest.java | 130 +++++
.../gremlin/structure/io/IoCustomTest.java | 5 +-
.../tinkerpop/gremlin/structure/io/IoTest.java | 6 +-
hadoop-gremlin/conf/hadoop-script.properties | 2 +
.../computer/AbstractHadoopGraphComputer.java | 31 ++
.../traversal/step/sideEffect/HadoopIoStep.java | 129 ++++-
.../structure/io/script/ScriptRecordReader.java | 17 +-
.../structure/io/script/ScriptRecordWriter.java | 13 +
.../structure/io/util/ConfigValueParser.java | 61 +++
.../structure/io/util/OlapClassLoadingPolicy.java | 190 +++++++
.../structure/io/util/OlapConfigKeyPolicy.java | 114 ++++
.../step/sideEffect/HadoopIoStepTest.java | 407 ++++++++++++++
.../io/script/ScriptRecordReaderTest.java | 83 +++
.../io/script/ScriptRecordReaderWriterTest.java | 3 +
.../io/script/ScriptRecordWriterTest.java | 86 +++
.../structure/io/util/ConfigValueParserTest.java | 62 +++
.../io/util/OlapClassLoadingPolicyTest.java | 202 +++++++
.../structure/io/util/OlapConfigKeyPolicyTest.java | 137 +++++
pom.xml | 55 ++
.../spark/process/computer/SparkGraphComputer.java | 91 +++-
.../SparkGraphComputerConfigSecurityTest.java | 281 ++++++++++
.../SparkHadoopIoSecurityIntegrateTest.java | 145 +++++
.../SparkLocalClusterIoSecurityIntegrateTest.java | 315 +++++++++++
.../computer/SystemPropertyProbeVertexProgram.java | 122 +++++
.../tinkergraph/structure/TinkerIoRegistryV2.java | 27 +-
.../tinkergraph/structure/TinkerIoRegistryV3.java | 27 +-
.../tinkergraph/structure/TinkerIoRegistryV4.java | 27 +-
.../TinkerGraphGraphSONSerializerV2Test.java | 36 ++
.../TinkerGraphGraphSONSerializerV3Test.java | 107 ++++
.../TinkerGraphGraphSONSerializerV4Test.java | 107 ++++
105 files changed, 7545 insertions(+), 259 deletions(-)
create mode 100644 .beads/issues.jsonl
create mode 100755 bin/beads-commits.py
create mode 100644 docs/src/data/air-routes.asciidoc
create mode 100644 docs/src/data/grateful-dead.asciidoc
create mode 100644 docs/src/data/index.asciidoc
create mode 100644 docs/src/data/modern.asciidoc
create mode 100644 docs/src/data/the-crew.asciidoc
create mode 100644 docs/src/data/the-zoo.asciidoc
create mode 100644 docs/tinkeradoc-extension/src/main/java/org/apache/tinkerpop/tinkeradoc/GraphCatalog.java
create mode 100644 docs/tinkeradoc-extension/src/test/java/org/apache/tinkerpop/tinkeradoc/GraphCatalogTest.java
create mode 100644 gremlin-core/src/test/java/com/example/gadget/GraphSONTestGadgets.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/map/VertexProgramStepTest.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategyTest.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/HaltedTraverserStrategyTest.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/finalization/MatchAlgorithmStrategyTest.java
copy gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/graphson/{GraphSONTokensTest.java => GraphSONMapperBuilderTest.java} (58%)
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/graphson/PGraphSONSerializerTest.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/graphson/TraversalStrategyProxyJacksonDeserializerTest.java
create mode 100644 gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/structure/io/util/IoRegistryHelperTest.java
create mode 100644 gremlin-server/src/test/java/org/apache/tinkerpop/gremlin/server/handler/HttpBasicAuthorizationHandlerTest.java
create mode 100644 hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/ConfigValueParser.java
create mode 100644 hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/OlapClassLoadingPolicy.java
create mode 100644 hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/OlapConfigKeyPolicy.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStepTest.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderTest.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriterTest.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/ConfigValueParserTest.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/OlapClassLoadingPolicyTest.java
create mode 100644 hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/util/OlapConfigKeyPolicyTest.java
create mode 100644 spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputerConfigSecurityTest.java
create mode 100644 spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopIoSecurityIntegrateTest.java
create mode 100644 spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkLocalClusterIoSecurityIntegrateTest.java
create mode 100644 spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SystemPropertyProbeVertexProgram.java
create mode 100644 tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphGraphSONSerializerV3Test.java
create mode 100644 tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphGraphSONSerializerV4Test.java
(tinkerpop) 18/20: Reject corrupt TinkerStorage frames with a clear error
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 72a09a4db3428b63dcc66c1f777889e37faf747e
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 16:36:57 2026 +0000
Reject corrupt TinkerStorage frames with a clear error
Surface an unknown value type code as a Corrupt storage IOException instead of
an opaque NullPointerException, matching the codec's other integrity checks.
Add decode-path tests for the four malformed-frame cases that clear CRC framing
but are internally invalid: unknown op code, dictionary append gap, dictionary
id redefinition, and unknown value type code.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 5 +-
.../structure/storage/GraphBinaryStorageTest.java | 69 ++++++++++++++++++++++
2 files changed, 73 insertions(+), 1 deletion(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 7c6d0cff77..e99e3c8192 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -391,7 +391,10 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
@SuppressWarnings({"unchecked", "rawtypes"})
private Object readScalar(final ByteBufferBuffer buf) throws IOException {
- final DataType dataType = DataType.get(Byte.toUnsignedInt(buf.readByte()));
+ final int code = Byte.toUnsignedInt(buf.readByte());
+ final DataType dataType = DataType.get(code);
+ if (dataType == null)
+ throw new IOException(String.format("Corrupt storage: unknown value type code 0x%02X", code));
if (dataType == DataType.UNSPECIFIED_NULL)
return null;
final TypeSerializer serializer = registry.getSerializer(dataType);
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index f5347f0920..bcb8605ba2 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -22,15 +22,20 @@ import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
import org.junit.Test;
import java.io.DataInputStream;
import java.io.File;
+import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Files;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -503,6 +508,70 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
}
}
+ // --- decode-error paths -----------------------------------------------------------------------------------------
+ // A frame that clears CRC framing can still be internally malformed (a bug in an older writer, or a targeted flip
+ // that happens to keep the checksum valid). The codec must reject each such frame with a clear IOException rather
+ // than a silent wrong answer or an opaque runtime crash. These drive decodeFrame directly with hand-built frames;
+ // values below 128 are single-byte varints, so the frames are written as literal bytes.
+
+ private static final byte OP_DEL_VERTEX = 2;
+ private static final byte OP_DICT_APPEND = 5;
+
+ /** Decode one frame payload against a fresh codec, returning nothing — the maps are throwaway. */
+ private static void decode(final GraphBinaryStorage codec, final byte[] frame) throws IOException {
+ final Map<Object, DetachedVertex> vertices = new HashMap<>();
+ final Map<Object, DetachedEdge> edges = new HashMap<>();
+ codec.decodeFrame(frame, vertices, edges);
+ }
+
+ @Test
+ public void shouldRejectFrameWithUnknownOpCode() {
+ // entryCount=1, op=99 (no such op)
+ try {
+ decode(new GraphBinaryStorage(), new byte[]{ 1, 99 });
+ fail("expected an IOException for an unknown op code");
+ } catch (final IOException expected) {
+ assertTrue(expected.getMessage(), expected.getMessage().contains("Unknown storage op code"));
+ }
+ }
+
+ @Test
+ public void shouldRejectDictionaryAppendGap() {
+ // entryCount=1, OP_DICT_APPEND, id=5 into an empty dictionary (expected next id is 0) -> gap
+ try {
+ decode(new GraphBinaryStorage(), new byte[]{ 1, OP_DICT_APPEND, 5, 1, (byte) 'x' });
+ fail("expected an IOException for a dictionary append gap");
+ } catch (final IOException expected) {
+ assertTrue(expected.getMessage(), expected.getMessage().contains("dictionary append gap"));
+ }
+ }
+
+ @Test
+ public void shouldRejectDictionaryRedefinition() {
+ // append id 0 = "a", then re-append id 0 = "b" on the same codec -> redefinition (a mismatch, not an
+ // idempotent re-append of the same string)
+ final GraphBinaryStorage codec = new GraphBinaryStorage();
+ try {
+ decode(codec, new byte[]{ 1, OP_DICT_APPEND, 0, 1, (byte) 'a' });
+ decode(codec, new byte[]{ 1, OP_DICT_APPEND, 0, 1, (byte) 'b' });
+ fail("expected an IOException for a redefined dictionary id");
+ } catch (final IOException expected) {
+ assertTrue(expected.getMessage(), expected.getMessage().contains("redefined"));
+ }
+ }
+
+ @Test
+ public void shouldRejectUnknownValueTypeCode() {
+ // OP_DEL_VERTEX reads a scalar id; feed it a value type code (0xFF) no serializer claims. Without a guard this
+ // is an opaque NullPointerException; the codec must surface it as corruption instead.
+ try {
+ decode(new GraphBinaryStorage(), new byte[]{ 1, OP_DEL_VERTEX, (byte) 0xFF });
+ fail("expected an IOException for an unknown value type code");
+ } catch (final IOException expected) {
+ assertTrue(expected.getMessage(), expected.getMessage().contains("unknown value type code"));
+ }
+ }
+
private static String rootMessage(final Throwable t) {
Throwable cur = t;
while (cur.getCause() != null && cur.getCause() != cur)
(tinkerpop) 12/20: Extract AbstractLogStorage base from GraphBinaryStorage
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 5f331196c9e458fc0cd3a28c24b0e5ae16228897
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 14:16:14 2026 +0000
Extract AbstractLogStorage base from GraphBinaryStorage
Move the codec-agnostic log-structured machinery — file layout, VERSION marker,
CRC framing, replay fold, SyncMode durability, and crash-safe + threshold
compaction — into a new abstract AbstractLogStorage. GraphBinaryStorage now
supplies only the element codec through four hooks (encodeCommit, decodeFrame,
writeSnapshot, beginReplay). Pure refactor: on-disk format and behavior are
unchanged, so a new codec can be built by overriding the hooks alone.
Assisted-by: Claude Code:claude-opus-4-8
---
...hBinaryStorage.java => AbstractLogStorage.java} | 442 +++++++-----------
.../structure/storage/GraphBinaryStorage.java | 512 ++-------------------
2 files changed, 202 insertions(+), 752 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
similarity index 63%
copy from tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
copy to tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
index 2aaa7643cc..b2bdf249f5 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
@@ -19,20 +19,15 @@
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
import org.apache.commons.configuration2.Configuration;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader;
-import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter;
-import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry;
import org.apache.tinkerpop.gremlin.structure.util.Attachable;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
+import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
@@ -42,7 +37,6 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.channels.FileChannel;
import java.nio.file.AtomicMoveNotSupportedException;
@@ -51,40 +45,24 @@ import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.zip.CRC32;
/**
- * A durable {@link TinkerStorage} engine that persists a {@code TinkerStorageGraph} as an append-only commit log
- * ("write-ahead log") serialized with GraphBinary. On each committed transaction the changeset is appended to
- * {@code log.gbin} as a single record; on open the optional {@code snapshot.gbin} is read followed by the log, with the
- * folded result re-applied to the in-memory graph. {@link #compact(AbstractTinkerGraph)} rewrites the snapshot from the
- * current committed state and truncates the log.
+ * Log-structured durable storage machinery shared by {@link TinkerStorage} engines, independent of how an element is
+ * encoded. It persists a {@code TinkerStorageGraph} as an append-only commit log ({@code log.gbin}) plus an optional
+ * folded {@code snapshot.gbin}; on open the snapshot is read followed by the log, last-write-wins per element id.
* <p/>
- * On commit the appended record is made durable according to the configured {@link SyncMode}: {@link SyncMode#COMMIT}
- * (default) {@code fsync}s so the commit survives an OS crash or power loss, while {@link SyncMode#OS} only flushes to
- * the operating system.
+ * This base owns everything that is not the element codec: the on-disk file layout, the single-source-of-truth
+ * {@code VERSION} marker, length+CRC frame framing (which tells an interrupted trailing append apart from genuine
+ * corruption), the replay fold loop, durability via {@link SyncMode}, and crash-safe atomic compaction with a
+ * size threshold. Concrete engines supply only the codec through {@link #encodeCommit}, {@link #decodeFrame},
+ * {@link #writeSnapshot}, and (optionally) {@link #beginReplay} for per-replay decode state.
* <p/>
- * On-disk compatibility: the store's format version is recorded once in a {@code VERSION} marker file (magic +
- * version), which is the single source of truth even when a store momentarily holds a snapshot and a log written at
- * different times. Individual files carry only the magic for identity/corruption detection. Opening a store whose
- * marker names an unsupported version fails loudly — records are never misread across a format change. The engine
- * keeps this simple by not attempting in-place migration: the supported path across an incompatible format bump is to
- * export via {@code g.io()} before upgrading. Additive changes should prefer new record opcodes; unknown opcodes are
- * a hard error (a durable store must not silently drop records it cannot parse), so a genuinely incompatible change
- * bumps the version.
- * <p/>
- * The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
- * <p/>
- * Known limitation (write amplification): a commit records each changed element in full — a single property change on
- * a large element rewrites the whole element to the log. Elements are typically small and automatic compaction bounds
- * the resulting log growth, so this is accepted rather than mitigated with per-property deltas, which would complicate
- * the {@link TinkerStorageMutation} contract and the replay fold. The snapshot, by contrast, is streamed one element
- * at a time so compaction never holds a second full copy of the graph in heap.
+ * The in-memory graph remains authoritative (write-through); this machinery does not support graphs larger than memory.
*/
-public final class GraphBinaryStorage implements TinkerStorage {
+public abstract class AbstractLogStorage implements TinkerStorage {
/**
* Magic bytes ("TGSB" — TinkerGraph Storage Binary) at the start of every storage file, so a file can be
@@ -101,8 +79,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
static final byte FORMAT_VERSION = 1;
/**
- * Store-level version marker file, holding {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}. It is
- * the single source of truth for the store's format version.
+ * Store-level version marker file, holding {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}.
*/
static final String VERSION_FILE = "VERSION";
@@ -112,27 +89,19 @@ public final class GraphBinaryStorage implements TinkerStorage {
*/
static final int HEADER_SIZE = MAGIC.length;
- private static final byte OP_PUT_VERTEX = 1;
- private static final byte OP_DEL_VERTEX = 2;
- private static final byte OP_PUT_EDGE = 3;
- private static final byte OP_DEL_EDGE = 4;
-
static final String SNAPSHOT_FILE = "snapshot.gbin";
static final String LOG_FILE = "log.gbin";
- private final GraphBinaryWriter writer = new GraphBinaryWriter(TypeSerializerRegistry.INSTANCE);
- private final GraphBinaryReader reader = new GraphBinaryReader(TypeSerializerRegistry.INSTANCE);
+ /**
+ * Default automatic-compaction threshold: 64 MB of appended log since the last compaction.
+ */
+ static final long DEFAULT_COMPACT_THRESHOLD_BYTES = 64L * 1024 * 1024;
private File directory;
private File snapshotFile;
private File logFile;
private File versionFile;
- /**
- * Default automatic-compaction threshold: 64 MB of appended log since the last compaction.
- */
- static final long DEFAULT_COMPACT_THRESHOLD_BYTES = 64L * 1024 * 1024;
-
private DataOutputStream logOut;
private FileOutputStream logFos;
private SyncMode syncMode = SyncMode.COMMIT;
@@ -140,11 +109,42 @@ public final class GraphBinaryStorage implements TinkerStorage {
private long logBytesSinceCompaction = 0;
private boolean closed = false;
+ // ----------------------------------------------------------------------------------------- codec hooks
+
+ /**
+ * Encode a committing transaction's changeset into a single record payload (the framing is added by the caller).
+ */
+ protected abstract byte[] encodeCommit(long txVersion,
+ Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException;
+
+ /**
+ * Decode one record payload, folding its puts and deletes into the supplied maps (last-write-wins per id).
+ */
+ protected abstract void decodeFrame(byte[] record,
+ Map<Object, DetachedVertex> vertices,
+ Map<Object, DetachedEdge> edges) throws IOException;
+
+ /**
+ * Write the entire current committed state of the graph to {@code out} as framed records (via {@link #writeFrame}),
+ * for compaction. The fixed {@link #MAGIC} header has already been written to {@code out}.
+ */
+ protected abstract void writeSnapshot(AbstractTinkerGraph graph, DataOutputStream out) throws IOException;
+
+ /**
+ * Reset any per-replay decode state (e.g. a dictionary) before a fold begins. Default is a no-op.
+ */
+ protected void beginReplay() {
+ // no-op by default
+ }
+
+ // ----------------------------------------------------------------------------------------- lifecycle
+
@Override
public void open(final AbstractTinkerGraph graph, final Configuration config) {
final String location = config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
if (null == location)
- throw new IllegalStateException(String.format("%s must be set to use the GraphBinary storage engine",
+ throw new IllegalStateException(String.format("%s must be set to use a durable storage engine",
TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION));
this.directory = new File(location);
this.snapshotFile = new File(directory, SNAPSHOT_FILE);
@@ -159,68 +159,9 @@ public final class GraphBinaryStorage implements TinkerStorage {
establishStoreVersion();
}
- /**
- * Read and validate the store-level version marker, or create it for a new store. This is the single source of
- * truth for the store's format version: a marker naming an unsupported version, or bad magic, fails the open
- * loudly rather than risking a misread. The supported path across an incompatible format bump is to export the
- * graph via {@code g.io()} before upgrading.
- */
- private void establishStoreVersion() {
- // an existing store (has a snapshot or log) written before the marker existed is treated as version 1
- final boolean storeHasData = snapshotFile.exists() || logFile.exists();
- if (!versionFile.exists()) {
- if (storeHasData && FORMAT_VERSION != 1)
- throw new IllegalStateException(String.format(
- "Storage location %s has data but no version marker; cannot confirm it is format version %d",
- directory, FORMAT_VERSION));
- writeStoreVersion();
- return;
- }
- try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(versionFile)))) {
- final byte[] magic = new byte[MAGIC.length];
- readFully(in, magic);
- if (!Arrays.equals(magic, MAGIC))
- throw new IOException(String.format("%s is not a TinkerGraph storage version marker (bad magic)", versionFile));
- final byte version = in.readByte();
- if (version != FORMAT_VERSION)
- throw new IOException(String.format(
- "Unsupported storage format version %d at %s (this build writes %d); export via g.io() before upgrading",
- version, directory, FORMAT_VERSION));
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not read storage version marker %s", versionFile), ex);
- }
- }
-
- /**
- * Write the store-level version marker ({@link #MAGIC} + {@link #FORMAT_VERSION}) durably.
- */
- private void writeStoreVersion() {
- try (final FileOutputStream fos = new FileOutputStream(versionFile);
- final DataOutputStream out = new DataOutputStream(fos)) {
- out.write(MAGIC);
- out.writeByte(FORMAT_VERSION);
- out.flush();
- fos.getFD().sync();
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not write storage version marker %s", versionFile), ex);
- }
- }
-
- /**
- * Ensure the backing directory exists, creating it if necessary. Called on open and again before writing a
- * snapshot, since {@code close()} may be invoked more than once and the directory may have been removed in between.
- */
- private void ensureDirectory() {
- if (directory.exists()) {
- if (!directory.isDirectory())
- throw new IllegalStateException(String.format("Storage location %s exists but is not a directory", directory));
- } else if (!directory.mkdirs()) {
- throw new IllegalStateException(String.format("Could not create storage directory %s", directory));
- }
- }
-
@Override
public void replay(final AbstractTinkerGraph graph) {
+ beginReplay();
// Fold snapshot then log into final state: last write per id wins, deletes remove.
final Map<Object, DetachedVertex> vertices = new LinkedHashMap<>();
final Map<Object, DetachedEdge> edges = new LinkedHashMap<>();
@@ -242,84 +183,13 @@ public final class GraphBinaryStorage implements TinkerStorage {
graph.tx().commit();
}
- /**
- * Read every record in a file, folding puts and deletes into the supplied maps.
- */
- private void foldRecords(final File file, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) {
- final long fileLength = file.length();
- try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(file)))) {
- long remaining = readAndVerifyHeader(in, file, fileLength);
- while (true) {
- final byte[] record = readFrame(in, remaining);
- if (record == null)
- break;
- // account for the header (length + crc) and payload just consumed
- remaining -= 2L * Integer.BYTES + record.length;
- applyRecord(record, vertices, edges);
- }
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not read storage file %s", file), ex);
- }
- }
-
- /**
- * Read and validate the per-file header (magic only), returning the number of record bytes that follow it. An
- * empty file (freshly created, no header yet) is treated as having no records. The store's format version is
- * validated once against the {@link #VERSION_FILE} marker in {@link #open}, not here.
- */
- private long readAndVerifyHeader(final DataInputStream in, final File file, final long fileLength) throws IOException {
- if (fileLength == 0)
- return 0;
- if (fileLength < HEADER_SIZE)
- throw new IOException(String.format("Corrupt storage file %s: shorter than its %d-byte header", file, HEADER_SIZE));
- final byte[] magic = new byte[MAGIC.length];
- readFully(in, magic);
- if (!Arrays.equals(magic, MAGIC))
- throw new IOException(String.format("%s is not a TinkerGraph storage file (bad magic)", file));
- return fileLength - HEADER_SIZE;
- }
-
- private void applyRecord(final byte[] record, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) throws IOException {
- // the format version is validated once per file in readAndVerifyHeader, so records no longer repeat it
- final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
- buffer.readLong(); // txVersion, retained for diagnostics/future use
- final int entryCount = buffer.readInt();
- for (int i = 0; i < entryCount; i++) {
- final byte op = buffer.readByte();
- switch (op) {
- case OP_PUT_VERTEX: {
- final Vertex v = reader.read(buffer);
- vertices.put(v.id(), (DetachedVertex) v);
- break;
- }
- case OP_DEL_VERTEX: {
- final Object id = reader.read(buffer);
- vertices.remove(id);
- break;
- }
- case OP_PUT_EDGE: {
- final Edge e = reader.read(buffer);
- edges.put(e.id(), (DetachedEdge) e);
- break;
- }
- case OP_DEL_EDGE: {
- final Object id = reader.read(buffer);
- edges.remove(id);
- break;
- }
- default:
- throw new IOException("Unknown storage op code: " + op);
- }
- }
- }
-
@Override
public void persist(final long txVersion,
final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) {
ensureLogOpen();
try {
- final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges);
+ final byte[] frame = encodeCommit(txVersion, changedVertices, changedEdges);
writeFrame(logOut, frame);
logBytesSinceCompaction += 2L * Integer.BYTES + frame.length; // length + crc prefixes + payload
} catch (IOException ex) {
@@ -327,39 +197,6 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
}
- /**
- * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
- * serialized element (put) or the serialized id (delete). The format version lives in the file header, not the
- * record.
- */
- private byte[] encodeRecord(final long txVersion,
- final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
- final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(txVersion);
- buffer.writeInt(changedVertices.size() + changedEdges.size());
- for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
- if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_VERTEX);
- writer.write(m.id(), buffer);
- } else {
- buffer.writeByte(OP_PUT_VERTEX);
- // detach to a stable form independent of the transactional element
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
- }
- }
- for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
- if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_EDGE);
- writer.write(m.id(), buffer);
- } else {
- buffer.writeByte(OP_PUT_EDGE);
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
- }
- }
- return buffer.toWrittenArray();
- }
-
@Override
public void flush() {
if (closed)
@@ -428,66 +265,96 @@ public final class GraphBinaryStorage implements TinkerStorage {
compact(graph);
}
+ @Override
+ public void close() {
+ closeLog();
+ closed = true;
+ }
+
+ // ----------------------------------------------------------------------------------------- version marker
+
/**
- * Atomically move {@code source} onto {@code target}, replacing any existing target. Falls back to a non-atomic
- * replacing move on filesystems that do not support atomic moves.
+ * Read and validate the store-level version marker, or create it for a new store. This is the single source of
+ * truth for the store's format version: a marker naming an unsupported version, or bad magic, fails the open
+ * loudly rather than risking a misread.
*/
- private static void atomicMove(final File source, final File target) throws IOException {
- try {
- Files.move(source.toPath(), target.toPath(),
- StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
- } catch (AtomicMoveNotSupportedException anse) {
- Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ private void establishStoreVersion() {
+ final boolean storeHasData = snapshotFile.exists() || logFile.exists();
+ if (!versionFile.exists()) {
+ if (storeHasData && FORMAT_VERSION != 1)
+ throw new IllegalStateException(String.format(
+ "Storage location %s has data but no version marker; cannot confirm it is format version %d",
+ directory, FORMAT_VERSION));
+ writeStoreVersion();
+ return;
+ }
+ try (final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(versionFile)))) {
+ final byte[] magic = new byte[MAGIC.length];
+ readFully(in, magic);
+ if (!Arrays.equals(magic, MAGIC))
+ throw new IOException(String.format("%s is not a TinkerGraph storage version marker (bad magic)", versionFile));
+ final byte version = in.readByte();
+ if (version != FORMAT_VERSION)
+ throw new IOException(String.format(
+ "Unsupported storage format version %d at %s (this build writes %d); export via g.io() before upgrading",
+ version, directory, FORMAT_VERSION));
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not read storage version marker %s", versionFile), ex);
}
}
- /**
- * fsync the storage directory so that recent namespace changes (a rename into place, a file deletion) are durable.
- * A directory fsync is required because those operations only update the directory entry, which the earlier file
- * fsync does not cover.
- */
- private void syncDirectory() {
- try (final FileChannel dirChannel = FileChannel.open(directory.toPath(), StandardOpenOption.READ)) {
- dirChannel.force(true);
+ private void writeStoreVersion() {
+ try (final FileOutputStream fos = new FileOutputStream(versionFile);
+ final DataOutputStream out = new DataOutputStream(fos)) {
+ out.write(MAGIC);
+ out.writeByte(FORMAT_VERSION);
+ out.flush();
+ fos.getFD().sync();
} catch (IOException ex) {
- // some platforms (notably Windows) cannot open a directory as a channel; the atomic rename is the
- // durability guarantee there, so treat inability to sync the directory as non-fatal
+ throw new UncheckedIOException(String.format("Could not write storage version marker %s", versionFile), ex);
}
}
- /**
- * Write the entire current committed state of the graph to {@code out} as a stream of single-element put records,
- * one frame per vertex and per edge. Each frame is an ordinary put record (see {@link #encodeRecord}) with an
- * entry count of one, so {@link #foldRecords} reconstructs the graph from these frames exactly as it would from a
- * commit log. Writing one element at a time keeps peak memory bounded to a single element rather than materializing
- * the whole graph as one byte array, so a snapshot never needs to hold a second full copy of the graph in heap.
- */
- private void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
- final Iterator<Vertex> vertexIterator = graph.vertices();
- while (vertexIterator.hasNext())
- writeElementFrame(out, OP_PUT_VERTEX, vertexIterator.next());
- final Iterator<Edge> edgeIterator = graph.edges();
- while (edgeIterator.hasNext())
- writeElementFrame(out, OP_PUT_EDGE, edgeIterator.next());
+ private void ensureDirectory() {
+ if (directory.exists()) {
+ if (!directory.isDirectory())
+ throw new IllegalStateException(String.format("Storage location %s exists but is not a directory", directory));
+ } else if (!directory.mkdirs()) {
+ throw new IllegalStateException(String.format("Could not create storage directory %s", directory));
+ }
}
- /**
- * Encode a single element as a one-entry put record and write it as a framed record to {@code out}. Only one
- * element's bytes are held in memory at a time.
- */
- private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(0L); // snapshot records have no single tx version
- buffer.writeInt(1);
- buffer.writeByte(op);
- writer.write(DetachedFactory.detach(element, true), buffer);
- writeFrame(out, buffer.toWrittenArray());
+ // ----------------------------------------------------------------------------------------- fold / framing
+
+ private void foldRecords(final File file, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) {
+ final long fileLength = file.length();
+ try (final DataInputStream in = new DataInputStream(new BufferedInputStream(new FileInputStream(file)))) {
+ long remaining = readAndVerifyHeader(in, file, fileLength);
+ while (true) {
+ final byte[] record = readFrame(in, remaining);
+ if (record == null)
+ break;
+ remaining -= 2L * Integer.BYTES + record.length;
+ decodeFrame(record, vertices, edges);
+ }
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not read storage file %s", file), ex);
+ }
}
- @Override
- public void close() {
- closeLog();
- closed = true;
+ /**
+ * Read and validate the per-file header (magic only), returning the number of record bytes that follow it.
+ */
+ private long readAndVerifyHeader(final DataInputStream in, final File file, final long fileLength) throws IOException {
+ if (fileLength == 0)
+ return 0;
+ if (fileLength < HEADER_SIZE)
+ throw new IOException(String.format("Corrupt storage file %s: shorter than its %d-byte header", file, HEADER_SIZE));
+ final byte[] magic = new byte[MAGIC.length];
+ readFully(in, magic);
+ if (!Arrays.equals(magic, MAGIC))
+ throw new IOException(String.format("%s is not a TinkerGraph storage file (bad magic)", file));
+ return fileLength - HEADER_SIZE;
}
private void ensureLogOpen() {
@@ -505,14 +372,6 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
}
- /**
- * Write the per-file header ({@link #MAGIC}) at the start of a storage file. The format version is recorded once
- * per store in the {@link #VERSION_FILE} marker, not per file.
- */
- private static void writeHeader(final DataOutputStream out) throws IOException {
- out.write(MAGIC);
- }
-
private void closeLog() {
if (logOut != null) {
try {
@@ -527,12 +386,19 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
}
+ /**
+ * Write the per-file header ({@link #MAGIC}) at the start of a storage file.
+ */
+ private static void writeHeader(final DataOutputStream out) throws IOException {
+ out.write(MAGIC);
+ }
+
/**
* Write a framed record: a 4-byte big-endian payload length, a 4-byte CRC32 of the payload, then the payload.
* The checksum lets a reader tell a bit-flip inside a complete frame (corruption) from a short final frame left
- * by an interrupted append (truncation).
+ * by an interrupted append (truncation). Available to codec subclasses writing per-element snapshot frames.
*/
- private static void writeFrame(final DataOutputStream out, final byte[] payload) throws IOException {
+ protected static void writeFrame(final DataOutputStream out, final byte[] payload) throws IOException {
final CRC32 crc = new CRC32();
crc.update(payload);
out.writeInt(payload.length);
@@ -541,14 +407,9 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Read a framed record, or return {@code null} at end of the readable log. A frame that is only partially present
- * — the file ends inside the header or payload — is treated as an interrupted trailing append (truncation) and
- * ends reading so earlier committed records still load. A frame that is fully present but whose stored CRC does
- * not match its payload is genuine corruption and is raised, rather than silently dropping it and everything
- * after it.
- *
- * @param remaining bytes left in the file at the current position; used to distinguish a short trailing frame
- * (truncation) from a complete frame, and to bound the payload allocation against a garbage length
+ * Read a framed record, or return {@code null} at end of the readable log. A frame only partially present is
+ * treated as an interrupted trailing append (truncation) and ends reading; a fully-present frame whose stored CRC
+ * does not match is genuine corruption and is raised.
*/
private static byte[] readFrame(final DataInputStream in, final long remaining) throws IOException {
if (remaining == 0)
@@ -588,4 +449,29 @@ public final class GraphBinaryStorage implements TinkerStorage {
off += read;
}
}
+
+ /**
+ * Atomically move {@code source} onto {@code target}, replacing any existing target. Falls back to a non-atomic
+ * replacing move on filesystems that do not support atomic moves.
+ */
+ private static void atomicMove(final File source, final File target) throws IOException {
+ try {
+ Files.move(source.toPath(), target.toPath(),
+ StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (AtomicMoveNotSupportedException anse) {
+ Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ /**
+ * fsync the storage directory so that recent namespace changes (a rename into place, a file deletion) are durable.
+ */
+ private void syncDirectory() {
+ try (final FileChannel dirChannel = FileChannel.open(directory.toPath(), StandardOpenOption.READ)) {
+ dirChannel.force(true);
+ } catch (IOException ex) {
+ // some platforms (notably Windows) cannot open a directory as a channel; the atomic rename is the
+ // durability guarantee there, so treat inability to sync the directory as non-fatal
+ }
+ }
}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 2aaa7643cc..a5f3fd1c5c 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -18,65 +18,29 @@
*/
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
-import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter;
import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry;
-import org.apache.tinkerpop.gremlin.structure.util.Attachable;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.UncheckedIOException;
-import java.nio.channels.FileChannel;
-import java.nio.file.AtomicMoveNotSupportedException;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.nio.file.StandardOpenOption;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
-import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.zip.CRC32;
/**
- * A durable {@link TinkerStorage} engine that persists a {@code TinkerStorageGraph} as an append-only commit log
- * ("write-ahead log") serialized with GraphBinary. On each committed transaction the changeset is appended to
- * {@code log.gbin} as a single record; on open the optional {@code snapshot.gbin} is read followed by the log, with the
- * folded result re-applied to the in-memory graph. {@link #compact(AbstractTinkerGraph)} rewrites the snapshot from the
- * current committed state and truncates the log.
- * <p/>
- * On commit the appended record is made durable according to the configured {@link SyncMode}: {@link SyncMode#COMMIT}
- * (default) {@code fsync}s so the commit survives an OS crash or power loss, while {@link SyncMode#OS} only flushes to
- * the operating system.
- * <p/>
- * On-disk compatibility: the store's format version is recorded once in a {@code VERSION} marker file (magic +
- * version), which is the single source of truth even when a store momentarily holds a snapshot and a log written at
- * different times. Individual files carry only the magic for identity/corruption detection. Opening a store whose
- * marker names an unsupported version fails loudly — records are never misread across a format change. The engine
- * keeps this simple by not attempting in-place migration: the supported path across an incompatible format bump is to
- * export via {@code g.io()} before upgrading. Additive changes should prefer new record opcodes; unknown opcodes are
- * a hard error (a durable store must not silently drop records it cannot parse), so a genuinely incompatible change
- * bumps the version.
- * <p/>
- * The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
+ * The GraphBinary {@link TinkerStorage} codec on top of {@link AbstractLogStorage}. The base owns the durable
+ * log-structured machinery (file layout, {@code VERSION} marker, CRC framing, replay fold, {@link SyncMode}
+ * durability, crash-safe and threshold compaction); this class supplies only how an element is encoded and decoded,
+ * using the GraphBinary serializers ({@link GraphBinaryWriter}/{@link GraphBinaryReader}).
* <p/>
* Known limitation (write amplification): a commit records each changed element in full — a single property change on
* a large element rewrites the whole element to the log. Elements are typically small and automatic compaction bounds
@@ -84,203 +48,53 @@ import java.util.zip.CRC32;
* the {@link TinkerStorageMutation} contract and the replay fold. The snapshot, by contrast, is streamed one element
* at a time so compaction never holds a second full copy of the graph in heap.
*/
-public final class GraphBinaryStorage implements TinkerStorage {
-
- /**
- * Magic bytes ("TGSB" — TinkerGraph Storage Binary) at the start of every storage file, so a file can be
- * identified as one written by this engine (and a foreign or corrupt file rejected) before any record is read.
- */
- static final byte[] MAGIC = { 'T', 'G', 'S', 'B' };
-
- /**
- * On-disk format version of the store. Recorded once per store in the {@link #VERSION_FILE} marker rather than in
- * every file, so a store has a single unambiguous version even when it momentarily holds a snapshot and a log
- * written at different times. A future format bump is detected against this marker so an older store is rejected
- * (never silently misread); the supported migration path is to export via {@code g.io()} before upgrading.
- */
- static final byte FORMAT_VERSION = 1;
-
- /**
- * Store-level version marker file, holding {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}. It is
- * the single source of truth for the store's format version.
- */
- static final String VERSION_FILE = "VERSION";
-
- /**
- * Bytes of the per-file header: just {@link #MAGIC}. The format version lives in the store-level
- * {@link #VERSION_FILE}, not in each file.
- */
- static final int HEADER_SIZE = MAGIC.length;
+public final class GraphBinaryStorage extends AbstractLogStorage {
private static final byte OP_PUT_VERTEX = 1;
private static final byte OP_DEL_VERTEX = 2;
private static final byte OP_PUT_EDGE = 3;
private static final byte OP_DEL_EDGE = 4;
- static final String SNAPSHOT_FILE = "snapshot.gbin";
- static final String LOG_FILE = "log.gbin";
-
private final GraphBinaryWriter writer = new GraphBinaryWriter(TypeSerializerRegistry.INSTANCE);
private final GraphBinaryReader reader = new GraphBinaryReader(TypeSerializerRegistry.INSTANCE);
- private File directory;
- private File snapshotFile;
- private File logFile;
- private File versionFile;
-
/**
- * Default automatic-compaction threshold: 64 MB of appended log since the last compaction.
+ * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
+ * serialized element (put) or the serialized id (delete).
*/
- static final long DEFAULT_COMPACT_THRESHOLD_BYTES = 64L * 1024 * 1024;
-
- private DataOutputStream logOut;
- private FileOutputStream logFos;
- private SyncMode syncMode = SyncMode.COMMIT;
- private long compactThresholdBytes = DEFAULT_COMPACT_THRESHOLD_BYTES;
- private long logBytesSinceCompaction = 0;
- private boolean closed = false;
-
@Override
- public void open(final AbstractTinkerGraph graph, final Configuration config) {
- final String location = config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
- if (null == location)
- throw new IllegalStateException(String.format("%s must be set to use the GraphBinary storage engine",
- TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION));
- this.directory = new File(location);
- this.snapshotFile = new File(directory, SNAPSHOT_FILE);
- this.logFile = new File(directory, LOG_FILE);
- this.versionFile = new File(directory, VERSION_FILE);
- this.syncMode = SyncMode.fromConfigValue(config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, null));
- this.compactThresholdBytes = config.getLong(
- TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, DEFAULT_COMPACT_THRESHOLD_BYTES);
- // seed the counter with any pre-existing log so a graph reopened with a large log still compacts promptly
- this.logBytesSinceCompaction = logFile.exists() ? logFile.length() : 0;
- ensureDirectory();
- establishStoreVersion();
- }
-
- /**
- * Read and validate the store-level version marker, or create it for a new store. This is the single source of
- * truth for the store's format version: a marker naming an unsupported version, or bad magic, fails the open
- * loudly rather than risking a misread. The supported path across an incompatible format bump is to export the
- * graph via {@code g.io()} before upgrading.
- */
- private void establishStoreVersion() {
- // an existing store (has a snapshot or log) written before the marker existed is treated as version 1
- final boolean storeHasData = snapshotFile.exists() || logFile.exists();
- if (!versionFile.exists()) {
- if (storeHasData && FORMAT_VERSION != 1)
- throw new IllegalStateException(String.format(
- "Storage location %s has data but no version marker; cannot confirm it is format version %d",
- directory, FORMAT_VERSION));
- writeStoreVersion();
- return;
- }
- try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(versionFile)))) {
- final byte[] magic = new byte[MAGIC.length];
- readFully(in, magic);
- if (!Arrays.equals(magic, MAGIC))
- throw new IOException(String.format("%s is not a TinkerGraph storage version marker (bad magic)", versionFile));
- final byte version = in.readByte();
- if (version != FORMAT_VERSION)
- throw new IOException(String.format(
- "Unsupported storage format version %d at %s (this build writes %d); export via g.io() before upgrading",
- version, directory, FORMAT_VERSION));
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not read storage version marker %s", versionFile), ex);
- }
- }
-
- /**
- * Write the store-level version marker ({@link #MAGIC} + {@link #FORMAT_VERSION}) durably.
- */
- private void writeStoreVersion() {
- try (final FileOutputStream fos = new FileOutputStream(versionFile);
- final DataOutputStream out = new DataOutputStream(fos)) {
- out.write(MAGIC);
- out.writeByte(FORMAT_VERSION);
- out.flush();
- fos.getFD().sync();
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not write storage version marker %s", versionFile), ex);
- }
- }
-
- /**
- * Ensure the backing directory exists, creating it if necessary. Called on open and again before writing a
- * snapshot, since {@code close()} may be invoked more than once and the directory may have been removed in between.
- */
- private void ensureDirectory() {
- if (directory.exists()) {
- if (!directory.isDirectory())
- throw new IllegalStateException(String.format("Storage location %s exists but is not a directory", directory));
- } else if (!directory.mkdirs()) {
- throw new IllegalStateException(String.format("Could not create storage directory %s", directory));
+ protected byte[] encodeCommit(final long txVersion,
+ final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
+ final ByteBufferBuffer buffer = new ByteBufferBuffer();
+ buffer.writeLong(txVersion);
+ buffer.writeInt(changedVertices.size() + changedEdges.size());
+ for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
+ if (m.isDeleted()) {
+ buffer.writeByte(OP_DEL_VERTEX);
+ writer.write(m.id(), buffer);
+ } else {
+ buffer.writeByte(OP_PUT_VERTEX);
+ // detach to a stable form independent of the transactional element
+ writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ }
}
- }
-
- @Override
- public void replay(final AbstractTinkerGraph graph) {
- // Fold snapshot then log into final state: last write per id wins, deletes remove.
- final Map<Object, DetachedVertex> vertices = new LinkedHashMap<>();
- final Map<Object, DetachedEdge> edges = new LinkedHashMap<>();
-
- if (snapshotFile.exists())
- foldRecords(snapshotFile, vertices, edges);
- if (logFile.exists())
- foldRecords(logFile, vertices, edges);
-
- if (vertices.isEmpty() && edges.isEmpty())
- return;
-
- // Attach vertices first so edges can find their endpoints, then commit once.
- for (final DetachedVertex v : vertices.values())
- v.attach(Attachable.Method.getOrCreate(graph));
- for (final DetachedEdge e : edges.values())
- e.attach(Attachable.Method.getOrCreate(graph));
-
- graph.tx().commit();
- }
-
- /**
- * Read every record in a file, folding puts and deletes into the supplied maps.
- */
- private void foldRecords(final File file, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) {
- final long fileLength = file.length();
- try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(file)))) {
- long remaining = readAndVerifyHeader(in, file, fileLength);
- while (true) {
- final byte[] record = readFrame(in, remaining);
- if (record == null)
- break;
- // account for the header (length + crc) and payload just consumed
- remaining -= 2L * Integer.BYTES + record.length;
- applyRecord(record, vertices, edges);
+ for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
+ if (m.isDeleted()) {
+ buffer.writeByte(OP_DEL_EDGE);
+ writer.write(m.id(), buffer);
+ } else {
+ buffer.writeByte(OP_PUT_EDGE);
+ writer.write(DetachedFactory.detach(m.element(), true), buffer);
}
- } catch (IOException ex) {
- throw new UncheckedIOException(String.format("Could not read storage file %s", file), ex);
}
+ return buffer.toWrittenArray();
}
- /**
- * Read and validate the per-file header (magic only), returning the number of record bytes that follow it. An
- * empty file (freshly created, no header yet) is treated as having no records. The store's format version is
- * validated once against the {@link #VERSION_FILE} marker in {@link #open}, not here.
- */
- private long readAndVerifyHeader(final DataInputStream in, final File file, final long fileLength) throws IOException {
- if (fileLength == 0)
- return 0;
- if (fileLength < HEADER_SIZE)
- throw new IOException(String.format("Corrupt storage file %s: shorter than its %d-byte header", file, HEADER_SIZE));
- final byte[] magic = new byte[MAGIC.length];
- readFully(in, magic);
- if (!Arrays.equals(magic, MAGIC))
- throw new IOException(String.format("%s is not a TinkerGraph storage file (bad magic)", file));
- return fileLength - HEADER_SIZE;
- }
-
- private void applyRecord(final byte[] record, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) throws IOException {
- // the format version is validated once per file in readAndVerifyHeader, so records no longer repeat it
+ @Override
+ protected void decodeFrame(final byte[] record,
+ final Map<Object, DetachedVertex> vertices,
+ final Map<Object, DetachedEdge> edges) throws IOException {
final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
buffer.readLong(); // txVersion, retained for diagnostics/future use
final int entryCount = buffer.readInt();
@@ -313,156 +127,12 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
}
- @Override
- public void persist(final long txVersion,
- final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
- final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) {
- ensureLogOpen();
- try {
- final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges);
- writeFrame(logOut, frame);
- logBytesSinceCompaction += 2L * Integer.BYTES + frame.length; // length + crc prefixes + payload
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not append transaction to storage log", ex);
- }
- }
-
/**
- * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
- * serialized element (put) or the serialized id (delete). The format version lives in the file header, not the
- * record.
+ * Stream the current committed state as one framed put record per vertex and per edge, so peak memory is bounded
+ * to a single element rather than materializing the whole graph as one byte array.
*/
- private byte[] encodeRecord(final long txVersion,
- final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
- final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(txVersion);
- buffer.writeInt(changedVertices.size() + changedEdges.size());
- for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
- if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_VERTEX);
- writer.write(m.id(), buffer);
- } else {
- buffer.writeByte(OP_PUT_VERTEX);
- // detach to a stable form independent of the transactional element
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
- }
- }
- for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
- if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_EDGE);
- writer.write(m.id(), buffer);
- } else {
- buffer.writeByte(OP_PUT_EDGE);
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
- }
- }
- return buffer.toWrittenArray();
- }
-
- @Override
- public void flush() {
- if (closed)
- return;
- if (logOut != null) {
- try {
- // flush the JVM buffer into the OS page cache; durable against a JVM process crash
- logOut.flush();
- // in COMMIT mode also force the OS page cache to the device, so an acknowledged commit is durable
- // against an OS crash or power loss. OS mode stops at the flush above and accepts that weaker guarantee.
- if (syncMode == SyncMode.COMMIT)
- logFos.getFD().sync();
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not flush storage log", ex);
- }
- }
- }
-
- @Override
- public void compact(final AbstractTinkerGraph graph) {
- if (closed)
- return;
- // Write a fresh snapshot of the current committed state, then truncate the log. This must be crash-safe: at
- // no point may a crash leave the store without a readable snapshot-or-log covering the committed state.
- // Ordering is write-tmp -> fsync tmp -> atomically rename tmp over the snapshot -> fsync dir (the rename is
- // now durable) -> delete the log -> fsync dir. The old snapshot is only ever replaced by an atomic rename, so
- // a crash at any step leaves either the old (snapshot + log) or the new (snapshot) intact — never neither.
- closeLog();
- ensureDirectory();
- final File tmp = new File(directory, SNAPSHOT_FILE + ".tmp");
- try (final FileOutputStream fos = new FileOutputStream(tmp);
- final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) {
- writeHeader(out);
- writeSnapshot(graph, out);
- out.flush();
- // force the snapshot's bytes to the device before it is renamed into place
- fos.getFD().sync();
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not write storage snapshot", ex);
- }
-
- try {
- // atomically replace the snapshot; no delete-then-rename window where the snapshot is briefly absent
- atomicMove(tmp, snapshotFile);
- // fsync the directory so the rename survives a crash before we touch the log
- syncDirectory();
-
- // truncate the log now that the snapshot durably reflects the committed state
- if (logFile.exists() && !logFile.delete())
- throw new IOException("Could not truncate storage log " + logFile);
- // fsync the directory again so the log's removal is durable
- syncDirectory();
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not finalize storage snapshot", ex);
- }
-
- // the log is now empty; the accumulated state lives in the snapshot
- logBytesSinceCompaction = 0;
- }
-
@Override
- public void maybeCompact(final AbstractTinkerGraph graph) {
- if (closed || compactThresholdBytes <= 0)
- return;
- if (logBytesSinceCompaction >= compactThresholdBytes)
- compact(graph);
- }
-
- /**
- * Atomically move {@code source} onto {@code target}, replacing any existing target. Falls back to a non-atomic
- * replacing move on filesystems that do not support atomic moves.
- */
- private static void atomicMove(final File source, final File target) throws IOException {
- try {
- Files.move(source.toPath(), target.toPath(),
- StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
- } catch (AtomicMoveNotSupportedException anse) {
- Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
- }
- }
-
- /**
- * fsync the storage directory so that recent namespace changes (a rename into place, a file deletion) are durable.
- * A directory fsync is required because those operations only update the directory entry, which the earlier file
- * fsync does not cover.
- */
- private void syncDirectory() {
- try (final FileChannel dirChannel = FileChannel.open(directory.toPath(), StandardOpenOption.READ)) {
- dirChannel.force(true);
- } catch (IOException ex) {
- // some platforms (notably Windows) cannot open a directory as a channel; the atomic rename is the
- // durability guarantee there, so treat inability to sync the directory as non-fatal
- }
- }
-
- /**
- * Write the entire current committed state of the graph to {@code out} as a stream of single-element put records,
- * one frame per vertex and per edge. Each frame is an ordinary put record (see {@link #encodeRecord}) with an
- * entry count of one, so {@link #foldRecords} reconstructs the graph from these frames exactly as it would from a
- * commit log. Writing one element at a time keeps peak memory bounded to a single element rather than materializing
- * the whole graph as one byte array, so a snapshot never needs to hold a second full copy of the graph in heap.
- */
- private void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
+ protected void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
final Iterator<Vertex> vertexIterator = graph.vertices();
while (vertexIterator.hasNext())
writeElementFrame(out, OP_PUT_VERTEX, vertexIterator.next());
@@ -472,8 +142,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Encode a single element as a one-entry put record and write it as a framed record to {@code out}. Only one
- * element's bytes are held in memory at a time.
+ * Encode a single element as a one-entry put record and write it as a framed record to {@code out}.
*/
private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
final ByteBufferBuffer buffer = new ByteBufferBuffer();
@@ -483,109 +152,4 @@ public final class GraphBinaryStorage implements TinkerStorage {
writer.write(DetachedFactory.detach(element, true), buffer);
writeFrame(out, buffer.toWrittenArray());
}
-
- @Override
- public void close() {
- closeLog();
- closed = true;
- }
-
- private void ensureLogOpen() {
- if (logOut == null) {
- try {
- final boolean freshFile = !logFile.exists() || logFile.length() == 0;
- // retain the FileOutputStream so flush() can reach its FileDescriptor for fsync
- logFos = new FileOutputStream(logFile, true);
- logOut = new DataOutputStream(new BufferedOutputStream(logFos));
- if (freshFile)
- writeHeader(logOut);
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not open storage log for append", ex);
- }
- }
- }
-
- /**
- * Write the per-file header ({@link #MAGIC}) at the start of a storage file. The format version is recorded once
- * per store in the {@link #VERSION_FILE} marker, not per file.
- */
- private static void writeHeader(final DataOutputStream out) throws IOException {
- out.write(MAGIC);
- }
-
- private void closeLog() {
- if (logOut != null) {
- try {
- logOut.flush();
- logOut.close();
- } catch (IOException ex) {
- throw new UncheckedIOException("Could not close storage log", ex);
- } finally {
- logOut = null;
- logFos = null;
- }
- }
- }
-
- /**
- * Write a framed record: a 4-byte big-endian payload length, a 4-byte CRC32 of the payload, then the payload.
- * The checksum lets a reader tell a bit-flip inside a complete frame (corruption) from a short final frame left
- * by an interrupted append (truncation).
- */
- private static void writeFrame(final DataOutputStream out, final byte[] payload) throws IOException {
- final CRC32 crc = new CRC32();
- crc.update(payload);
- out.writeInt(payload.length);
- out.writeInt((int) crc.getValue());
- out.write(payload);
- }
-
- /**
- * Read a framed record, or return {@code null} at end of the readable log. A frame that is only partially present
- * — the file ends inside the header or payload — is treated as an interrupted trailing append (truncation) and
- * ends reading so earlier committed records still load. A frame that is fully present but whose stored CRC does
- * not match its payload is genuine corruption and is raised, rather than silently dropping it and everything
- * after it.
- *
- * @param remaining bytes left in the file at the current position; used to distinguish a short trailing frame
- * (truncation) from a complete frame, and to bound the payload allocation against a garbage length
- */
- private static byte[] readFrame(final DataInputStream in, final long remaining) throws IOException {
- if (remaining == 0)
- return null; // clean end of file, exactly on a frame boundary
- if (remaining < 2L * Integer.BYTES)
- return null; // not even a full header left — interrupted append
-
- final int length = in.readInt();
- final int storedCrc = in.readInt();
- if (length < 0)
- throw new IOException("Corrupt storage frame: negative payload length " + length);
- if ((long) length > remaining - 2L * Integer.BYTES)
- return null; // frame claims more bytes than remain — truncated trailing append
-
- final byte[] payload = new byte[length];
- try {
- readFully(in, payload);
- } catch (EOFException eof) {
- return null; // partial trailing payload from an interrupted append
- }
-
- final CRC32 crc = new CRC32();
- crc.update(payload);
- if ((int) crc.getValue() != storedCrc)
- throw new IOException(String.format(
- "Corrupt storage frame: CRC mismatch (stored %08x, computed %08x) in a fully-present %d-byte record",
- storedCrc, (int) crc.getValue(), length));
- return payload;
- }
-
- private static void readFully(final InputStream in, final byte[] dst) throws IOException {
- int off = 0;
- while (off < dst.length) {
- final int read = in.read(dst, off, dst.length - off);
- if (read < 0)
- throw new EOFException();
- off += read;
- }
- }
}
(tinkerpop) 19/20: Document TinkerStorageGraph single-writer stance; enforce meta-property conflict
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit eb33eb3153b1edd853e03f3429c791f9195a39bb
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Thu Aug 20 15:07:03 2026 +0000
Document TinkerStorageGraph single-writer stance; enforce meta-property conflict
Add a comment explaining why supportsConcurrentAccess() is false: a persistent
TinkerStorageGraph is a single-writer store (DirectoryLock), and the feature
denotes multiple connections/instances sharing the same data, not the
intra-instance multi-thread transaction access the graph already provides.
Re-enable the stale commented-out fail() in the concurrent meta-property test
and strengthen its assertion so it proves the losing transaction rolled back,
turning a decorative test into a real guard for conflict detection.
Assisted-by: Claude Code:claude-opus-4-8
---
.../gremlin/tinkergraph/structure/TinkerStorageGraph.java | 6 ++++++
.../gremlin/tinkergraph/structure/TinkerStorageGraphTest.java | 8 +++++---
2 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
index 0eced39af1..803cc27dcf 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
@@ -545,6 +545,12 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
private TinkerGraphGraphFeatures() {
}
+ /**
+ * A persistent {@link TinkerStorageGraph} is a single-writer store: {@code DirectoryLock} permits only one
+ * graph instance to open a given storage directory at a time. This feature denotes multiple connections /
+ * instances sharing the same data — not the intra-instance, multi-thread transaction access that the
+ * thread-local {@link TinkerTransaction} already provides — so it is {@code false}.
+ */
@Override
public boolean supportsConcurrentAccess() {
return false;
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraphTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraphTest.java
index af5e79124a..67bb6bffdd 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraphTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraphTest.java
@@ -1415,14 +1415,16 @@ public class TinkerStorageGraphTest {
try {
gtx.tx().commit();
- //fail("should throw TransactionException");
+ fail("should throw TransactionException");
} catch (TransactionException e) {
}
- // should be only 1 vertex with updated property
+ // tx2 committed first and wins; tx1 conflicted and rolled back, so its meta1 was never persisted
assertEquals(1L, (long) gtx.V().count().next());
- assertEquals("tx2", gtx.V(v1.id()).properties("test").values("meta1", "meta2").next());
+ assertEquals(1L, (long) gtx.V(v1.id()).properties("test").properties().count().next());
+ assertEquals("tx2", gtx.V(v1.id()).properties("test").values("meta2").next());
+ assertFalse(gtx.V(v1.id()).properties("test").values("meta1").hasNext());
}
@Test
(tinkerpop) 08/20: Add integrity checks to TinkerStorageGraph storage format
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 5d569762236d2309b5c1e08d1a97451e6d36fea0
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 17:29:56 2026 +0000
Add integrity checks to TinkerStorageGraph storage format
Storage files could silently misread corrupt data: a bit-flip inside a frame
went undetected, and a garbage frame length was indistinguishable from a
truncated trailing append. Each frame now carries a CRC32, and every file
starts with a magic + version header. On read, a complete frame with a bad
CRC or a file with bad magic / an unknown version fails loudly, while a short
trailing frame is still tolerated as an interrupted append. The payload
allocation is bounded by the remaining file bytes, so a corrupt length can no
longer trigger a huge allocation.
The per-record version byte is dropped in favor of the file header. This
changes the on-disk format, which is safe as the storage feature is unreleased.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 119 ++++++++++++++++-----
.../structure/storage/GraphBinaryStorageTest.java | 70 +++++++++++-
2 files changed, 159 insertions(+), 30 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 074b67bc34..715dc48c74 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -49,10 +49,12 @@ import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
+import java.util.zip.CRC32;
/**
* A durable {@link TinkerStorage} engine that persists a {@code TinkerStorageGraph} as an append-only commit log
@@ -76,10 +78,22 @@ import java.util.Map;
public final class GraphBinaryStorage implements TinkerStorage {
/**
- * Version byte prefixing every record, allowing the on-disk format to evolve.
+ * Magic bytes ("TGSB" — TinkerGraph Storage Binary) at the start of every storage file, so a file can be
+ * identified as one written by this engine (and a foreign or corrupt file rejected) before any record is read.
+ */
+ static final byte[] MAGIC = { 'T', 'G', 'S', 'B' };
+
+ /**
+ * On-disk format version, written once in each file's header after {@link #MAGIC}. A future format bump is
+ * detected here so old files can be rejected (or, later, migrated) rather than misread.
*/
static final byte FORMAT_VERSION = 1;
+ /**
+ * Bytes of the fixed file header: {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}.
+ */
+ static final int HEADER_SIZE = MAGIC.length + 1;
+
private static final byte OP_PUT_VERTEX = 1;
private static final byte OP_DEL_VERTEX = 2;
private static final byte OP_PUT_EDGE = 3;
@@ -164,16 +178,15 @@ public final class GraphBinaryStorage implements TinkerStorage {
* Read every record in a file, folding puts and deletes into the supplied maps.
*/
private void foldRecords(final File file, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) {
+ final long fileLength = file.length();
try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(file)))) {
+ long remaining = readAndVerifyHeader(in, file, fileLength);
while (true) {
- final byte[] record;
- try {
- record = readFrame(in);
- } catch (EOFException eof) {
- break;
- }
+ final byte[] record = readFrame(in, remaining);
if (record == null)
break;
+ // account for the header (length + crc) and payload just consumed
+ remaining -= 2L * Integer.BYTES + record.length;
applyRecord(record, vertices, edges);
}
} catch (IOException ex) {
@@ -181,11 +194,29 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
}
+ /**
+ * Read and validate the fixed file header, returning the number of record bytes that follow it. An empty file
+ * (freshly created, no header yet) is treated as having no records.
+ */
+ private long readAndVerifyHeader(final DataInputStream in, final File file, final long fileLength) throws IOException {
+ if (fileLength == 0)
+ return 0;
+ if (fileLength < HEADER_SIZE)
+ throw new IOException(String.format("Corrupt storage file %s: shorter than its %d-byte header", file, HEADER_SIZE));
+ final byte[] magic = new byte[MAGIC.length];
+ readFully(in, magic);
+ if (!Arrays.equals(magic, MAGIC))
+ throw new IOException(String.format("%s is not a TinkerGraph storage file (bad magic)", file));
+ final byte version = in.readByte();
+ if (version != FORMAT_VERSION)
+ throw new IOException(String.format(
+ "Unsupported storage format version %d in %s (expected %d)", version, file, FORMAT_VERSION));
+ return fileLength - HEADER_SIZE;
+ }
+
private void applyRecord(final byte[] record, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) throws IOException {
+ // the format version is validated once per file in readAndVerifyHeader, so records no longer repeat it
final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
- final byte version = buffer.readByte();
- if (version != FORMAT_VERSION)
- throw new IOException(String.format("Unsupported storage record version %d (expected %d)", version, FORMAT_VERSION));
buffer.readLong(); // txVersion, retained for diagnostics/future use
final int entryCount = buffer.readInt();
for (int i = 0; i < entryCount; i++) {
@@ -225,21 +256,21 @@ public final class GraphBinaryStorage implements TinkerStorage {
try {
final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges);
writeFrame(logOut, frame);
- logBytesSinceCompaction += Integer.BYTES + frame.length; // length prefix + payload
+ logBytesSinceCompaction += 2L * Integer.BYTES + frame.length; // length + crc prefixes + payload
} catch (IOException ex) {
throw new UncheckedIOException("Could not append transaction to storage log", ex);
}
}
/**
- * Serialize a commit record: version byte, txVersion, entry count, then each entry as an op byte followed by
- * either the serialized element (put) or the serialized id (delete).
+ * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
+ * serialized element (put) or the serialized id (delete). The format version lives in the file header, not the
+ * record.
*/
private byte[] encodeRecord(final long txVersion,
final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeByte(FORMAT_VERSION);
buffer.writeLong(txVersion);
buffer.writeInt(changedVertices.size() + changedEdges.size());
for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
@@ -296,6 +327,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
final File tmp = new File(directory, SNAPSHOT_FILE + ".tmp");
try (final FileOutputStream fos = new FileOutputStream(tmp);
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) {
+ writeHeader(out);
writeSnapshot(graph, out);
out.flush();
// force the snapshot's bytes to the device before it is renamed into place
@@ -380,7 +412,6 @@ public final class GraphBinaryStorage implements TinkerStorage {
*/
private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeByte(FORMAT_VERSION);
buffer.writeLong(0L); // snapshot records have no single tx version
buffer.writeInt(1);
buffer.writeByte(op);
@@ -397,15 +428,26 @@ public final class GraphBinaryStorage implements TinkerStorage {
private void ensureLogOpen() {
if (logOut == null) {
try {
+ final boolean freshFile = !logFile.exists() || logFile.length() == 0;
// retain the FileOutputStream so flush() can reach its FileDescriptor for fsync
logFos = new FileOutputStream(logFile, true);
logOut = new DataOutputStream(new BufferedOutputStream(logFos));
+ if (freshFile)
+ writeHeader(logOut);
} catch (IOException ex) {
throw new UncheckedIOException("Could not open storage log for append", ex);
}
}
}
+ /**
+ * Write the fixed file header ({@link #MAGIC} + {@link #FORMAT_VERSION}) at the start of a storage file.
+ */
+ private static void writeHeader(final DataOutputStream out) throws IOException {
+ out.write(MAGIC);
+ out.writeByte(FORMAT_VERSION);
+ }
+
private void closeLog() {
if (logOut != null) {
try {
@@ -421,33 +463,54 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Write a length-prefixed frame: a 4-byte big-endian length followed by the payload.
+ * Write a framed record: a 4-byte big-endian payload length, a 4-byte CRC32 of the payload, then the payload.
+ * The checksum lets a reader tell a bit-flip inside a complete frame (corruption) from a short final frame left
+ * by an interrupted append (truncation).
*/
private static void writeFrame(final DataOutputStream out, final byte[] payload) throws IOException {
+ final CRC32 crc = new CRC32();
+ crc.update(payload);
out.writeInt(payload.length);
+ out.writeInt((int) crc.getValue());
out.write(payload);
}
/**
- * Read a length-prefixed frame, or return {@code null} on a clean end of file. A truncated final frame (from a
- * crash mid-append) is treated as end of file so earlier committed records still load.
+ * Read a framed record, or return {@code null} at end of the readable log. A frame that is only partially present
+ * — the file ends inside the header or payload — is treated as an interrupted trailing append (truncation) and
+ * ends reading so earlier committed records still load. A frame that is fully present but whose stored CRC does
+ * not match its payload is genuine corruption and is raised, rather than silently dropping it and everything
+ * after it.
+ *
+ * @param remaining bytes left in the file at the current position; used to distinguish a short trailing frame
+ * (truncation) from a complete frame, and to bound the payload allocation against a garbage length
*/
- private static byte[] readFrame(final DataInputStream in) throws IOException {
- final int length;
- try {
- length = in.readInt();
- } catch (EOFException eof) {
- return null;
- }
+ private static byte[] readFrame(final DataInputStream in, final long remaining) throws IOException {
+ if (remaining == 0)
+ return null; // clean end of file, exactly on a frame boundary
+ if (remaining < 2L * Integer.BYTES)
+ return null; // not even a full header left — interrupted append
+
+ final int length = in.readInt();
+ final int storedCrc = in.readInt();
if (length < 0)
- throw new IOException("Corrupt storage frame length: " + length);
+ throw new IOException("Corrupt storage frame: negative payload length " + length);
+ if ((long) length > remaining - 2L * Integer.BYTES)
+ return null; // frame claims more bytes than remain — truncated trailing append
+
final byte[] payload = new byte[length];
try {
readFully(in, payload);
} catch (EOFException eof) {
- // partial trailing frame from an interrupted append — stop here
- return null;
+ return null; // partial trailing payload from an interrupted append
}
+
+ final CRC32 crc = new CRC32();
+ crc.update(payload);
+ if ((int) crc.getValue() != storedCrc)
+ throw new IOException(String.format(
+ "Corrupt storage frame: CRC mismatch (stored %08x, computed %08x) in a fully-present %d-byte record",
+ storedCrc, (int) crc.getValue(), length));
return payload;
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 2d6f9952d5..34e67ab93d 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -33,6 +33,7 @@ import java.util.Iterator;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Runs the shared {@link AbstractTinkerStorageConformanceTest} suite against the {@link GraphBinaryStorage} engine and
@@ -135,12 +136,14 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
}
/**
- * Count the length-prefixed frames in a storage file: each frame is a 4-byte big-endian length followed by that
- * many payload bytes.
+ * Count the framed records in a storage file: a fixed header ({@code HEADER_SIZE} bytes) followed by frames of a
+ * 4-byte big-endian payload length, a 4-byte CRC, then that many payload bytes.
*/
private static int countFrames(final File file) throws Exception {
int frames = 0;
try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new java.io.FileInputStream(file)))) {
+ final long headerSkipped = in.skip(GraphBinaryStorage.HEADER_SIZE);
+ if (headerSkipped < GraphBinaryStorage.HEADER_SIZE) return 0;
while (true) {
final int len;
try {
@@ -148,6 +151,7 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
} catch (java.io.EOFException eof) {
break;
}
+ in.readInt(); // CRC
final long skipped = in.skip(len);
if (skipped < len) break;
frames++;
@@ -241,6 +245,68 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.close();
}
+ @Test
+ public void shouldFailOnCorruptFrameWithBadCrc() throws Exception {
+ TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ graph.addVertex(T.id, 1, "value", 1);
+ graph.tx().commit();
+ graph.addVertex(T.id, 2, "value", 2);
+ graph.tx().commit();
+ graph.tx().close();
+
+ // preserve the raw log, then reconstruct it with a bit flipped inside the first frame's payload — a complete
+ // frame whose CRC no longer matches (distinct from a short trailing frame, which is tolerated as truncation)
+ final File logFile = new File(location, GraphBinaryStorage.LOG_FILE);
+ final byte[] log = Files.readAllBytes(logFile.toPath());
+ graph.close();
+ // header, then first frame's 4-byte length + 4-byte CRC, then payload — flip the first payload byte
+ final int firstPayloadByte = GraphBinaryStorage.HEADER_SIZE + 2 * Integer.BYTES;
+ log[firstPayloadByte] ^= 0x01;
+ Files.deleteIfExists(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).toPath());
+ Files.write(logFile.toPath(), log);
+
+ try {
+ open();
+ fail("expected reopen to fail on a CRC mismatch");
+ } catch (Exception expected) {
+ assertTrue("cause should report corruption: " + rootMessage(expected),
+ rootMessage(expected).contains("CRC mismatch"));
+ }
+ }
+
+ @Test
+ public void shouldFailOnForeignFileWithBadMagic() throws Exception {
+ TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ graph.addVertex(T.id, 1);
+ graph.tx().commit();
+ graph.tx().close();
+ final File logFile = new File(location, GraphBinaryStorage.LOG_FILE);
+ final byte[] log = Files.readAllBytes(logFile.toPath());
+ graph.close();
+
+ // corrupt the magic so the file no longer identifies as a TinkerGraph storage file
+ log[0] ^= 0xFF;
+ Files.deleteIfExists(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).toPath());
+ Files.write(logFile.toPath(), log);
+
+ try {
+ open();
+ fail("expected reopen to fail on bad magic");
+ } catch (Exception expected) {
+ assertTrue("cause should report a bad storage file: " + rootMessage(expected),
+ rootMessage(expected).contains("not a TinkerGraph storage file"));
+ }
+ }
+
+ private static String rootMessage(final Throwable t) {
+ Throwable cur = t;
+ while (cur.getCause() != null && cur.getCause() != cur)
+ cur = cur.getCause();
+ return String.valueOf(cur.getMessage());
+ }
+
private static long countOf(final Iterator<?> it) {
long count = 0;
while (it.hasNext()) {
(tinkerpop) 14/20: Add opt-in vertex-property id persistence to TinkerStorageGraph storage
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 41423dedbe40eb3f3218729ba4ca3d31540ef234
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 14:37:00 2026 +0000
Add opt-in vertex-property id persistence to TinkerStorageGraph storage
Add gremlin.tinkergraph.storage.preserveVertexPropertyIds (default false). When
enabled, the GraphBinary codec persists each vertex-property id so it is stable
across reopen; by default those ids are regenerated on load to keep the store
smaller. A per-vertex-record flag makes each record self-describing, so a store
written with the option reopens correctly even if the reader's setting differs.
A new configureCodec hook lets the codec read its own config at open.
Also adds codec tests: dictionary growth across log commits, dictionary rewrite
on compaction, preserve-ids on/off, and a bytes/element size-regression guard
(well under the ~168 bytes/element the old whole-object format cost).
Assisted-by: Claude Code:claude-opus-4-8
---
.../gremlin/tinkergraph/structure/TinkerGraph.java | 8 ++
.../structure/storage/AbstractLogStorage.java | 8 ++
.../structure/storage/GraphBinaryStorage.java | 21 ++++
.../structure/storage/GraphBinaryStorageTest.java | 119 +++++++++++++++++++++
4 files changed, 156 insertions(+)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 65a02537c6..f011cd5202 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -75,6 +75,14 @@ public interface TinkerGraph extends Graph {
* set.
*/
String GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD = "gremlin.tinkergraph.storage.compactThreshold";
+ /**
+ * Whether a {@link TinkerStorageGraph} storage engine persists auto-generated vertex-property ids so they are
+ * stable across a close and reopen. Defaults to {@code false}: vertex-property ids are regenerated on load, which
+ * keeps the store smaller. Element and edge ids are always preserved regardless of this setting. Each record is
+ * self-describing, so a store written with this enabled reopens correctly even if the setting later differs. Only
+ * meaningful when {@link #GREMLIN_TINKERGRAPH_STORAGE} is set.
+ */
+ String GREMLIN_TINKERGRAPH_STORAGE_PRESERVE_VP_IDS = "gremlin.tinkergraph.storage.preserveVertexPropertyIds";
String GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES = "gremlin.tinkergraph.allowNullPropertyValues";
String GREMLIN_TINKERGRAPH_SERVICE = "gremlin.tinkergraph.service";
String GREMLIN_TINKERGRAPH_VERTEX_LABEL_CARDINALITY = "gremlin.tinkergraph.vertexLabelCardinality";
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
index b2bdf249f5..4c8eefeb08 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractLogStorage.java
@@ -155,10 +155,18 @@ public abstract class AbstractLogStorage implements TinkerStorage {
TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, DEFAULT_COMPACT_THRESHOLD_BYTES);
// seed the counter with any pre-existing log so a graph reopened with a large log still compacts promptly
this.logBytesSinceCompaction = logFile.exists() ? logFile.length() : 0;
+ configureCodec(config);
ensureDirectory();
establishStoreVersion();
}
+ /**
+ * Read any codec-specific configuration. Called once during {@link #open}. Default is a no-op.
+ */
+ protected void configureCodec(final Configuration config) {
+ // no-op by default
+ }
+
@Override
public void replay(final AbstractTinkerGraph graph) {
beginReplay();
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 1266dee705..7c6d0cff77 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
@@ -33,6 +34,7 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
import java.io.DataOutputStream;
@@ -84,6 +86,17 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
private final Map<String, Integer> keyToId = new HashMap<>();
private final List<String> idToKey = new ArrayList<>();
+ /**
+ * When true, persist auto-generated vertex-property ids so they are stable across reopen. Written per vertex
+ * record so a store reopens correctly regardless of the reader's setting.
+ */
+ private boolean preserveVertexPropertyIds = false;
+
+ @Override
+ protected void configureCodec(final Configuration config) {
+ this.preserveVertexPropertyIds = config.getBoolean(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_PRESERVE_VP_IDS, false);
+ }
+
@Override
protected void beginReplay() {
keyToId.clear();
@@ -213,6 +226,9 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
for (final String label : labels)
writeVarInt(buf, keyToId.get(label));
+ // self-describing flag: whether each value below carries a persisted vertex-property id
+ buf.writeByte(preserveVertexPropertyIds ? 1 : 0);
+
// group vertex properties by key so multi-properties (list/set) round-trip
final Map<String, List<VertexProperty<Object>>> groups = new LinkedHashMap<>();
final Iterator<VertexProperty<Object>> vps = v.properties();
@@ -227,6 +243,8 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
writeVarInt(buf, values.size());
for (final VertexProperty<Object> vp : values) {
writeScalar(buf, vp.value());
+ if (preserveVertexPropertyIds)
+ writeScalar(buf, vp.id());
final List<Property<Object>> metas = new ArrayList<>();
vp.properties().forEachRemaining(metas::add);
writeVarInt(buf, metas.size());
@@ -315,6 +333,7 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
labels.add(idToKey.get(readVarInt(buf)));
b.setLabels(labels);
}
+ final boolean hasVpIds = buf.readByte() != 0;
final int keyGroupCount = readVarInt(buf);
for (int g = 0; g < keyGroupCount; g++) {
final String key = idToKey.get(readVarInt(buf));
@@ -322,6 +341,8 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
for (int j = 0; j < valueCount; j++) {
final Object value = readScalar(buf);
final DetachedVertexProperty.Builder vpb = DetachedVertexProperty.build().setLabel(key).setValue(value);
+ if (hasVpIds)
+ vpb.setId(readScalar(buf));
final int metaCount = readVarInt(buf);
for (int m = 0; m < metaCount; m++) {
final String metaKey = idToKey.get(readVarInt(buf));
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index cbde701a22..f5347f0920 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
import org.junit.Test;
@@ -384,6 +385,124 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
}
}
+ @Test
+ public void shouldRegenerateVertexPropertyIdsByDefault() {
+ // default: vertex-property ids are not persisted; the property still round-trips (value + meta), the id is
+ // simply reassigned on load
+ TinkerStorageGraph graph = open();
+ try {
+ final VertexProperty<Object> vp = graph.addVertex(T.id, 1).property(VertexProperty.Cardinality.list, "name", "marko");
+ vp.property("since", 2010);
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ final VertexProperty<Object> vp = graph.vertices(1).next().<Object>properties("name").next();
+ assertEquals("marko", vp.value());
+ assertEquals(Integer.valueOf(2010), vp.<Integer>property("since").value());
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldPreserveVertexPropertyIdsWhenConfigured() {
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_PRESERVE_VP_IDS, true);
+ final Object vpId;
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ try {
+ final VertexProperty<Object> vp = graph.addVertex(T.id, 1).property(VertexProperty.Cardinality.list, "name", "marko");
+ vpId = vp.id();
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = TinkerStorageGraph.open(conf);
+ try {
+ final VertexProperty<Object> vp = graph.vertices(1).next().<Object>properties("name").next();
+ assertEquals("marko", vp.value());
+ assertEquals("vertex-property id should be preserved across reopen", vpId, vp.id());
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldReplayDictionaryGrowthAcrossLogCommits() throws Exception {
+ // each commit introduces a new property key, so the dictionary grows via OP_DICT_APPEND across successive log
+ // frames. Reopening from a log with no snapshot must rebuild the dictionary incrementally and resolve all refs.
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 0L); // keep the log, no auto-compaction
+ final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ for (int i = 0; i < 10; i++) {
+ graph.addVertex(T.id, i, "key" + i, i);
+ graph.tx().commit();
+ }
+ graph.tx().close();
+ final byte[] log = Files.readAllBytes(new File(location, GraphBinaryStorage.LOG_FILE).toPath());
+ graph.close();
+
+ // restore a log-only store (no snapshot) and reopen
+ Files.deleteIfExists(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).toPath());
+ Files.write(new File(location, GraphBinaryStorage.LOG_FILE).toPath(), log);
+ graph = TinkerStorageGraph.open(conf);
+ try {
+ assertEquals(10, countOf(graph.vertices()));
+ for (int i = 0; i < 10; i++)
+ assertEquals(Integer.valueOf(i), graph.vertices(i).next().value("key" + i));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRewriteDictionaryOnCompactionAndReopen() {
+ TinkerStorageGraph graph = open();
+ try {
+ for (int i = 0; i < 10; i++)
+ graph.addVertex(T.id, i, "key" + i, i);
+ graph.tx().commit();
+ graph.compact(); // writes a fresh full-dictionary snapshot header, then element frames
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ assertEquals(10, countOf(graph.vertices()));
+ for (int i = 0; i < 10; i++)
+ assertEquals(Integer.valueOf(i), graph.vertices(i).next().value("key" + i));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldStoreFewBytesPerElement() {
+ // regression guard: the dictionary-encoded format must stay well under the ~168 bytes/element the old
+ // whole-object format cost for a comparable graph (3 vertex props, 2 edge props, E=V).
+ final int vertexCount = 200;
+ final TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ try {
+ for (int i = 0; i < vertexCount; i++)
+ graph.addVertex(T.id, i, "name", "v" + i, "age", i % 100, "score", i * 1.5d);
+ for (int i = 0; i < vertexCount; i++)
+ graph.vertices(i).next().addEdge("knows", graph.vertices((i + 1) % vertexCount).next(),
+ T.id, 1_000_000 + i, "weight", i * 0.5d, "count", i % 7);
+ graph.tx().commit();
+ graph.compact();
+ final long bytes = new File(location, GraphBinaryStorage.SNAPSHOT_FILE).length();
+ final double perElement = (double) bytes / (2 * vertexCount);
+ assertTrue("expected well under 168 bytes/element (whole-object baseline), got " + perElement, perElement < 100.0);
+ } finally {
+ graph.close();
+ }
+ }
+
private static String rootMessage(final Throwable t) {
Throwable cur = t;
while (cur.getCause() != null && cur.getCause() != cur)
(tinkerpop) 09/20: Add crash-consistency and streaming-scale tests for TinkerStorageGraph storage
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 3f6b48c69b343ca543fe2d58200a11c8d7cd8636
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 19:41:10 2026 +0000
Add crash-consistency and streaming-scale tests for TinkerStorageGraph storage
Fills the storage test gaps left after the durability/integrity work.
StorageCrashConsistencyTest reconstructs the exact on-disk states a crash
leaves at each step of the write-ahead commit and compaction sequences —
durable-commit-without-snapshot, snapshot-plus-log, stray temp snapshot before
rename, and new-snapshot-with-log-not-yet-deleted — and asserts each reopens to
the correct graph, deterministically and without killing a JVM.
GraphBinaryStorageTest gains a scaled snapshot-streaming test (500 vertices +
499 edges must write exactly one frame per element) as a bounded-memory proxy,
and documents why real fsync/power-loss durability is out of unit-test scope
(needs OS-level fault injection).
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorageTest.java | 70 ++++++++-
.../storage/StorageCrashConsistencyTest.java | 173 +++++++++++++++++++++
2 files changed, 238 insertions(+), 5 deletions(-)
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 34e67ab93d..7daf2eb5ed 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -37,7 +37,14 @@ import static org.junit.Assert.fail;
/**
* Runs the shared {@link AbstractTinkerStorageConformanceTest} suite against the {@link GraphBinaryStorage} engine and
- * adds engine-specific tests for the on-disk log layout.
+ * adds engine-specific tests for the on-disk log layout, sync modes, auto-compaction, snapshot streaming, and
+ * corruption detection.
+ * <p/>
+ * Not covered here (deliberately): true {@code fsync} durability against OS crash or power loss. The {@code commit}
+ * vs. {@code os} sync-mode tests verify configuration and a graceful round-trip, but a JVM unit test cannot prove that
+ * an acknowledged commit survives a kernel crash — that needs OS-level fault injection (e.g. a FUSE layer that drops
+ * un-synced writes, or {@code dm-flakey}), which is out of scope. Crash-*consistency* of the file layout (as opposed
+ * to device-level durability) is covered deterministically by {@link StorageCrashConsistencyTest}.
*/
public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest {
@@ -135,6 +142,43 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
reopened.close();
}
+ @Test
+ public void shouldStreamSnapshotFrameByFrameAtScale() {
+ // Bounded-memory proxy for the streaming snapshot path: rather than measure heap (flaky, JVM-dependent), assert
+ // the observable streaming property holds at scale — a large graph is written as exactly one frame per element,
+ // never one whole-graph frame — and round-trips intact. This is the property that keeps compaction from
+ // materializing a second full copy of the graph in memory; it is not a hard OOM assertion.
+ final int vertexCount = 500;
+ final int edgeCount = 499;
+ final TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ try {
+ for (int i = 0; i < vertexCount; i++)
+ graph.addVertex(T.id, i, "value", i);
+ for (int i = 0; i < edgeCount; i++)
+ graph.vertices(i).next().addEdge("next", graph.vertices(i + 1).next(), T.id, 1_000_000 + i);
+ graph.tx().commit();
+ graph.compact();
+
+ try {
+ assertEquals(vertexCount + edgeCount, countFrames(new File(location, GraphBinaryStorage.SNAPSHOT_FILE)));
+ } catch (Exception ex) {
+ throw new RuntimeException(ex);
+ }
+ } finally {
+ graph.close();
+ }
+
+ final TinkerStorageGraph reopened = open();
+ try {
+ assertEquals(vertexCount, countOf(reopened.vertices()));
+ assertEquals(edgeCount, countOf(reopened.edges()));
+ assertEquals(Integer.valueOf(499), reopened.vertices(499).next().value("value"));
+ } finally {
+ reopened.close();
+ }
+ }
+
/**
* Count the framed records in a storage file: a fixed header ({@code HEADER_SIZE} bytes) followed by frames of a
* 4-byte big-endian payload length, a 4-byte CRC, then that many payload bytes.
@@ -142,8 +186,7 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
private static int countFrames(final File file) throws Exception {
int frames = 0;
try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new java.io.FileInputStream(file)))) {
- final long headerSkipped = in.skip(GraphBinaryStorage.HEADER_SIZE);
- if (headerSkipped < GraphBinaryStorage.HEADER_SIZE) return 0;
+ if (!skipFully(in, GraphBinaryStorage.HEADER_SIZE)) return 0;
while (true) {
final int len;
try {
@@ -152,14 +195,31 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
break;
}
in.readInt(); // CRC
- final long skipped = in.skip(len);
- if (skipped < len) break;
+ if (!skipFully(in, len)) break;
frames++;
}
}
return frames;
}
+ /**
+ * Skip exactly {@code n} bytes, reading in a loop because a single {@link DataInputStream#skip} may skip fewer.
+ * Returns false if EOF is reached first.
+ */
+ private static boolean skipFully(final DataInputStream in, final long n) throws Exception {
+ long left = n;
+ while (left > 0) {
+ final long s = in.skip(left);
+ if (s <= 0) {
+ if (in.read() < 0) return false; // genuine EOF
+ left -= 1;
+ } else {
+ left -= s;
+ }
+ }
+ return true;
+ }
+
@Test
public void shouldAutoCompactWhenLogExceedsThreshold() {
// a small threshold makes automatic compaction fire mid-run, without any explicit compact()/close()
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
new file mode 100644
index 0000000000..614cfdb0ba
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
@@ -0,0 +1,173 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.BaseConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Crash-consistency tests for {@link GraphBinaryStorage}. Rather than kill a JVM mid-operation — which is slow and
+ * non-deterministic — these reconstruct the exact on-disk states a crash would leave at each step of the two durable
+ * sequences (the write-ahead commit and compaction) and assert that reopening recovers the correct graph. The
+ * invariant under test: at no step may a crash leave the store unable to recover the last committed state.
+ */
+public class StorageCrashConsistencyTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private String location;
+ private File snapshotFile;
+ private File logFile;
+ private File tmpSnapshotFile;
+
+ @Before
+ public void setUp() throws Exception {
+ final File dir = tempFolder.newFolder("storage");
+ location = dir.getAbsolutePath();
+ snapshotFile = new File(dir, GraphBinaryStorage.SNAPSHOT_FILE);
+ logFile = new File(dir, GraphBinaryStorage.LOG_FILE);
+ tmpSnapshotFile = new File(dir, GraphBinaryStorage.SNAPSHOT_FILE + ".tmp");
+ }
+
+ private Configuration config() {
+ final Configuration conf = new BaseConfiguration();
+ conf.setProperty(Graph.GRAPH, TinkerStorageGraph.class.getName());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE, "graphbinary");
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, location);
+ // disable auto-compaction so tests control exactly when compaction happens
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 0L);
+ return conf;
+ }
+
+ private TinkerStorageGraph open() {
+ return TinkerStorageGraph.open(config());
+ }
+
+ /**
+ * Captured building blocks of valid on-disk files, used to assemble crash states: a snapshot holding {1}, a log
+ * holding a later commit of {2}, and a compacted snapshot holding {1,2}.
+ */
+ private byte[] snapshotV1;
+ private byte[] logV2;
+ private byte[] snapshotV12;
+
+ private void captureBuildingBlocks() throws IOException {
+ // snapshot holding vertex 1 (compaction on close folds the single commit into the snapshot)
+ TinkerStorageGraph g = open();
+ g.addVertex(T.id, 1, "value", 1);
+ g.tx().commit();
+ g.close();
+ snapshotV1 = Files.readAllBytes(snapshotFile.toPath());
+
+ // a valid log holding a later commit of vertex 2, captured before the closing compaction folds it away
+ g = open(); // replays {1} from the snapshot
+ g.addVertex(T.id, 2, "value", 2);
+ g.tx().commit();
+ g.tx().close();
+ logV2 = Files.readAllBytes(logFile.toPath());
+ g.close(); // compacts {1,2} into the snapshot
+ snapshotV12 = Files.readAllBytes(snapshotFile.toPath());
+
+ // reset the directory to a clean slate for the state under test
+ resetFiles();
+ }
+
+ private void resetFiles() throws IOException {
+ Files.deleteIfExists(snapshotFile.toPath());
+ Files.deleteIfExists(logFile.toPath());
+ Files.deleteIfExists(tmpSnapshotFile.toPath());
+ }
+
+ private void assertReopensTo(final int... expectedIds) {
+ final TinkerStorageGraph graph = open();
+ try {
+ assertEquals(expectedIds.length, countOf(graph.vertices()));
+ for (final int id : expectedIds)
+ assertEquals(Integer.valueOf(id), graph.vertices(id).next().value("value"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRecoverDurableCommitWithNoSnapshot() throws Exception {
+ // WAL guarantee: a commit whose frame was durably written to the log, with no compaction having run, must
+ // recover on reopen even though no snapshot exists.
+ captureBuildingBlocks();
+ Files.write(logFile.toPath(), logV2);
+ assertReopensTo(2);
+ }
+
+ @Test
+ public void shouldRecoverFromSnapshotPlusLog() throws Exception {
+ // steady pre-compaction state: a snapshot holding earlier commits and a log holding later ones. Reopen must
+ // fold snapshot-then-log into the union.
+ captureBuildingBlocks();
+ Files.write(snapshotFile.toPath(), snapshotV1);
+ Files.write(logFile.toPath(), logV2);
+ assertReopensTo(1, 2);
+ }
+
+ @Test
+ public void shouldIgnoreStrayTempSnapshotFromCrashBeforeRename() throws Exception {
+ // crash after writing snapshot.gbin.tmp but before the atomic rename: the old snapshot + log are intact and
+ // the stray .tmp must be ignored, so the last committed state still recovers.
+ captureBuildingBlocks();
+ Files.write(snapshotFile.toPath(), snapshotV1);
+ Files.write(logFile.toPath(), logV2);
+ Files.write(tmpSnapshotFile.toPath(), new byte[]{ 0x00, 0x01, 0x02, 0x03 }); // garbage half-written temp
+ assertReopensTo(1, 2);
+ }
+
+ @Test
+ public void shouldRecoverFromNewSnapshotWithLogNotYetDeleted() throws Exception {
+ // crash after the rename installed the new snapshot but before the log was truncated: snapshot holds {1,2}
+ // and the stale log still holds {2}. Folding snapshot-then-log is idempotent (last write per id wins), so the
+ // result is exactly {1,2} — never a lost or duplicated element.
+ captureBuildingBlocks();
+ Files.write(snapshotFile.toPath(), snapshotV12);
+ Files.write(logFile.toPath(), logV2);
+ assertReopensTo(1, 2);
+ }
+
+ private static long countOf(final Iterator<?> it) {
+ long count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count++;
+ }
+ return count;
+ }
+}
(tinkerpop) 07/20: Stream TinkerStorageGraph snapshots one element at a time
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 6840d08d2cf79e3292bca002f2564826d812d205
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 16:28:49 2026 +0000
Stream TinkerStorageGraph snapshots one element at a time
Compaction built the entire graph snapshot as a single in-heap byte array
(with a doubling buffer), so compacting a large graph needed a second full
copy of it in memory and risked OOM. The snapshot is now streamed one
element per framed record straight to the file, bounding peak memory to a
single element. The on-disk format is unchanged: each frame is an ordinary
single-entry put record that replay folds exactly as before.
Write amplification (a commit rewrites each changed element in full) is
documented as a known limitation rather than mitigated, since elements are
small and auto-compaction bounds log growth.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 51 +++++++++++-----------
.../structure/storage/GraphBinaryStorageTest.java | 51 ++++++++++++++++++++++
2 files changed, 77 insertions(+), 25 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index c9eecba9e5..074b67bc34 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -49,11 +49,9 @@ import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
/**
@@ -68,6 +66,12 @@ import java.util.Map;
* the operating system.
* <p/>
* The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
+ * <p/>
+ * Known limitation (write amplification): a commit records each changed element in full — a single property change on
+ * a large element rewrites the whole element to the log. Elements are typically small and automatic compaction bounds
+ * the resulting log growth, so this is accepted rather than mitigated with per-property deltas, which would complicate
+ * the {@link TinkerStorageMutation} contract and the replay fold. The snapshot, by contrast, is streamed one element
+ * at a time so compaction never holds a second full copy of the graph in heap.
*/
public final class GraphBinaryStorage implements TinkerStorage {
@@ -292,9 +296,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
final File tmp = new File(directory, SNAPSHOT_FILE + ".tmp");
try (final FileOutputStream fos = new FileOutputStream(tmp);
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) {
- final byte[] frame = encodeSnapshot(graph);
- if (frame.length > 0)
- writeFrame(out, frame);
+ writeSnapshot(graph, out);
out.flush();
// force the snapshot's bytes to the device before it is renamed into place
fos.getFD().sync();
@@ -357,34 +359,33 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Serialize the entire current committed state of the graph as a single put-only record.
+ * Write the entire current committed state of the graph to {@code out} as a stream of single-element put records,
+ * one frame per vertex and per edge. Each frame is an ordinary put record (see {@link #encodeRecord}) with an
+ * entry count of one, so {@link #foldRecords} reconstructs the graph from these frames exactly as it would from a
+ * commit log. Writing one element at a time keeps peak memory bounded to a single element rather than materializing
+ * the whole graph as one byte array, so a snapshot never needs to hold a second full copy of the graph in heap.
*/
- private byte[] encodeSnapshot(final AbstractTinkerGraph graph) throws IOException {
- final List<Vertex> vertexList = new ArrayList<>();
+ private void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
final Iterator<Vertex> vertexIterator = graph.vertices();
while (vertexIterator.hasNext())
- vertexList.add(vertexIterator.next());
- final List<Edge> edgeList = new ArrayList<>();
+ writeElementFrame(out, OP_PUT_VERTEX, vertexIterator.next());
final Iterator<Edge> edgeIterator = graph.edges();
while (edgeIterator.hasNext())
- edgeList.add(edgeIterator.next());
-
- if (vertexList.isEmpty() && edgeList.isEmpty())
- return new byte[0];
+ writeElementFrame(out, OP_PUT_EDGE, edgeIterator.next());
+ }
+ /**
+ * Encode a single element as a one-entry put record and write it as a framed record to {@code out}. Only one
+ * element's bytes are held in memory at a time.
+ */
+ private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
final ByteBufferBuffer buffer = new ByteBufferBuffer();
buffer.writeByte(FORMAT_VERSION);
- buffer.writeLong(0L); // snapshot has no single tx version
- buffer.writeInt(vertexList.size() + edgeList.size());
- for (final Vertex v : vertexList) {
- buffer.writeByte(OP_PUT_VERTEX);
- writer.write(DetachedFactory.detach(v, true), buffer);
- }
- for (final Edge e : edgeList) {
- buffer.writeByte(OP_PUT_EDGE);
- writer.write(DetachedFactory.detach(e, true), buffer);
- }
- return buffer.toWrittenArray();
+ buffer.writeLong(0L); // snapshot records have no single tx version
+ buffer.writeInt(1);
+ buffer.writeByte(op);
+ writer.write(DetachedFactory.detach(element, true), buffer);
+ writeFrame(out, buffer.toWrittenArray());
}
@Override
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 8113e03d18..2d6f9952d5 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -20,10 +20,12 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
import org.junit.Test;
+import java.io.DataInputStream;
import java.io.File;
import java.io.RandomAccessFile;
import java.nio.file.Files;
@@ -105,6 +107,55 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.close();
}
+ @Test
+ public void shouldStreamSnapshotAsOneFramePerElement() throws Exception {
+ final TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ final Vertex a = graph.addVertex(T.id, 1, "name", "a");
+ final Vertex b = graph.addVertex(T.id, 2, "name", "b");
+ final Vertex c = graph.addVertex(T.id, 3, "name", "c");
+ a.addEdge("knows", b, T.id, 10);
+ b.addEdge("knows", c, T.id, 11);
+ graph.tx().commit();
+ graph.compact();
+
+ // the snapshot must be written as one framed record per element (3 vertices + 2 edges = 5), rather than a
+ // single whole-graph frame, so compaction never buffers the entire graph in one array
+ final File snapshotFile = new File(location, GraphBinaryStorage.SNAPSHOT_FILE);
+ assertEquals(5, countFrames(snapshotFile));
+ graph.close();
+
+ // and the streamed snapshot must reopen to exactly the same graph
+ final TinkerStorageGraph reopened = open();
+ assertEquals(3, countOf(reopened.vertices()));
+ assertEquals(2, countOf(reopened.edges()));
+ assertEquals("a", reopened.vertices(1).next().value("name"));
+ assertEquals("knows", reopened.edges(10).next().label());
+ reopened.close();
+ }
+
+ /**
+ * Count the length-prefixed frames in a storage file: each frame is a 4-byte big-endian length followed by that
+ * many payload bytes.
+ */
+ private static int countFrames(final File file) throws Exception {
+ int frames = 0;
+ try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new java.io.FileInputStream(file)))) {
+ while (true) {
+ final int len;
+ try {
+ len = in.readInt();
+ } catch (java.io.EOFException eof) {
+ break;
+ }
+ final long skipped = in.skip(len);
+ if (skipped < len) break;
+ frames++;
+ }
+ }
+ return frames;
+ }
+
@Test
public void shouldAutoCompactWhenLogExceedsThreshold() {
// a small threshold makes automatic compaction fire mid-run, without any explicit compact()/close()
(tinkerpop) 05/20: Lock TinkerStorageGraph storage directory to a single writer
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit d09a3a53240bd1d6ce433a0862a6a97382c3c3e8
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 15:15:22 2026 +0000
Lock TinkerStorageGraph storage directory to a single writer
A persistent TinkerStorageGraph is a single-writer embedded store, but nothing
stopped a second graph — in the same JVM or another process — from opening the
same location and corrupting its log and snapshot. open() now takes an
exclusive OS advisory lock on a LOCK file in the storage directory, held for
the graph's lifetime and released on close(); a second open fails fast with a
clear error. An OS lock is used so the kernel releases it if the JVM dies.
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 2 +-
.../tinkergraph/structure/AbstractTinkerGraph.java | 13 +++
.../tinkergraph/structure/TinkerStorageGraph.java | 25 ++++-
.../structure/storage/DirectoryLock.java | 115 +++++++++++++++++++++
.../structure/storage/DirectoryLockTest.java | 101 ++++++++++++++++++
.../structure/storage/GraphBinaryStorageTest.java | 15 ++-
6 files changed, 261 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 5757e51ff8..b67d2ca09e 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,7 +28,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message.
* Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)*
* Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)*
-* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability.
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast.
* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
[[release-4-0-0-beta-3]]
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
index bb663ad0bd..17e359b94f 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
@@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComp
import org.apache.tinkerpop.gremlin.gql.GqlDeclarativeMatchStrategy;
import org.apache.tinkerpop.gremlin.tinkergraph.services.TinkerServiceRegistry;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.DefaultStorage;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.DirectoryLock;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorage;
import java.lang.reflect.InvocationTargetException;
@@ -85,6 +86,12 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
*/
protected TinkerStorage storage;
+ /**
+ * Exclusive lock on the storage directory, held for the graph's lifetime so no second graph — in this or another
+ * process — can open the same location and corrupt its files. {@code null} when the graph is purely in-memory.
+ */
+ protected DirectoryLock directoryLock;
+
/**
* Serializes the durable write of a committing transaction. TinkerGraph transactions lock only their own changed
* elements, so two commits touching disjoint elements run their commit paths concurrently; without this lock they
@@ -327,6 +334,12 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
storage.close();
} finally {
storageCommitLock.unlock();
+ // release the exclusive directory lock last, so the location is only reopenable once the engine has
+ // fully released its files
+ if (directoryLock != null) {
+ directoryLock.close();
+ directoryLock = null;
+ }
}
}
serviceRegistry.close();
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
index 409f618f25..0eced39af1 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
@@ -35,8 +35,10 @@ import org.apache.tinkerpop.gremlin.gql.GqlDeclarativeMatchStrategy;
import org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.strategy.optimization.TinkerGraphCountStrategy;
import org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.strategy.optimization.TinkerGraphStepStrategy;
import org.apache.tinkerpop.gremlin.tinkergraph.services.TinkerServiceRegistry;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.DirectoryLock;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import java.io.File;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
@@ -107,12 +109,25 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
serviceRegistry.registerService(instantiate(serviceClass)));
if (storage != null) {
- storage.open(this, configuration);
- loading = true;
+ // take an exclusive lock on the storage directory before the engine touches any files, so a second graph
+ // on the same location fails fast rather than corrupting it. The directory must exist to hold the lock.
+ final File dir = new File(graphLocation);
+ if (!dir.isDirectory() && !dir.mkdirs())
+ throw new IllegalStateException(String.format("Could not create storage directory %s", dir));
+ directoryLock = DirectoryLock.acquire(dir);
try {
- storage.replay(this);
- } finally {
- loading = false;
+ storage.open(this, configuration);
+ loading = true;
+ try {
+ storage.replay(this);
+ } finally {
+ loading = false;
+ }
+ } catch (RuntimeException | Error ex) {
+ // don't leak the lock if the engine fails to open or replay
+ directoryLock.close();
+ directoryLock = null;
+ throw ex;
}
}
}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLock.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLock.java
new file mode 100644
index 0000000000..061e4e3372
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLock.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.channels.OverlappingFileLockException;
+import java.nio.file.StandardOpenOption;
+
+/**
+ * An exclusive, whole-process lock on a {@code TinkerStorageGraph} storage directory. A persistent, transactional
+ * TinkerGraph is a single-writer embedded store: two graphs opened on the same directory — whether in one JVM or
+ * across processes — would both append to and compact the same files, corrupting them. This holds an OS advisory lock
+ * ({@link FileLock}) on a {@code LOCK} file in the directory for the lifetime of the graph, so a second open fails
+ * fast rather than silently corrupting data.
+ * <p/>
+ * An OS lock (rather than a mere marker file) is used so the kernel releases it automatically if the JVM dies,
+ * avoiding a stale lock that would wedge the store after a crash.
+ * <p/>
+ * Note: {@link FileLock} semantics are unreliable on some network filesystems (notably NFS); this guarantee holds on
+ * local filesystems.
+ */
+public final class DirectoryLock implements AutoCloseable {
+
+ static final String LOCK_FILE = "LOCK";
+
+ private final FileChannel channel;
+ private final FileLock lock;
+ private final File lockFile;
+
+ private DirectoryLock(final FileChannel channel, final FileLock lock, final File lockFile) {
+ this.channel = channel;
+ this.lock = lock;
+ this.lockFile = lockFile;
+ }
+
+ /**
+ * Acquire an exclusive lock on the {@code LOCK} file within {@code directory}.
+ *
+ * @param directory the storage directory, which must already exist
+ * @return the held lock, released by {@link #close()}
+ * @throws IllegalStateException if another graph (in this or another process) already holds the lock
+ */
+ public static DirectoryLock acquire(final File directory) {
+ final File lockFile = new File(directory, LOCK_FILE);
+ FileChannel channel = null;
+ try {
+ channel = FileChannel.open(lockFile.toPath(),
+ StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ final FileLock lock = channel.tryLock();
+ if (lock == null) {
+ channel.close();
+ throw lockedByAnother(directory, null);
+ }
+ return new DirectoryLock(channel, lock, lockFile);
+ } catch (OverlappingFileLockException ofle) {
+ // another graph in *this* JVM already holds (or is acquiring) the lock on this file
+ closeQuietly(channel);
+ throw lockedByAnother(directory, ofle);
+ } catch (IOException ex) {
+ closeQuietly(channel);
+ throw new UncheckedIOException(String.format("Could not acquire storage lock for %s", directory), ex);
+ }
+ }
+
+ private static IllegalStateException lockedByAnother(final File directory, final Throwable cause) {
+ return new IllegalStateException(String.format(
+ "Storage location %s is already in use by another TinkerStorageGraph (in this or another process); "
+ + "a persistent TinkerStorageGraph allows only a single writer", directory), cause);
+ }
+
+ private static void closeQuietly(final FileChannel channel) {
+ if (channel != null) {
+ try {
+ channel.close();
+ } catch (IOException ignored) {
+ // best effort on the failure path
+ }
+ }
+ }
+
+ /**
+ * Release the lock and close the channel. Idempotent-friendly: safe to call once per acquired lock.
+ */
+ @Override
+ public void close() {
+ try {
+ if (lock.isValid())
+ lock.release();
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not release storage lock %s", lockFile), ex);
+ } finally {
+ closeQuietly(channel);
+ }
+ }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLockTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLockTest.java
new file mode 100644
index 0000000000..6856dddbf2
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DirectoryLockTest.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.BaseConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileLock;
+import java.nio.file.StandardOpenOption;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Verifies that a {@link TinkerStorageGraph} takes an exclusive lock on its storage directory so a second opener on
+ * the same location fails fast rather than corrupting the store.
+ */
+public class DirectoryLockTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private String location;
+
+ @Before
+ public void setUp() throws Exception {
+ location = tempFolder.newFolder("storage").getAbsolutePath();
+ }
+
+ private Configuration config() {
+ final Configuration conf = new BaseConfiguration();
+ conf.setProperty(Graph.GRAPH, TinkerStorageGraph.class.getName());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE, "graphbinary");
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, location);
+ return conf;
+ }
+
+ @Test
+ public void shouldReleaseLockOnCloseAndAllowReopen() {
+ // sequential open/close must not leave the directory wedged
+ TinkerStorageGraph graph = TinkerStorageGraph.open(config());
+ graph.addVertex(T.id, 1);
+ graph.tx().commit();
+ graph.close();
+
+ graph = TinkerStorageGraph.open(config());
+ assertNotNull(graph.vertices(1).next());
+ graph.close();
+ }
+
+ @Test
+ public void shouldRejectSecondOpenWhileLocationIsLocked() throws Exception {
+ // simulate another process holding the directory lock by taking the OS lock on the LOCK file directly
+ final File dir = new File(location);
+ dir.mkdirs();
+ final File lockFile = new File(dir, DirectoryLock.LOCK_FILE);
+ try (final FileChannel channel = FileChannel.open(lockFile.toPath(),
+ StandardOpenOption.CREATE, StandardOpenOption.WRITE);
+ final FileLock held = channel.lock()) {
+ assertNotNull(held);
+ try {
+ TinkerStorageGraph.open(config());
+ fail("expected open to fail while the storage location is locked");
+ } catch (IllegalStateException expected) {
+ assertTrue("message should name the location: " + expected.getMessage(),
+ expected.getMessage().contains(location));
+ }
+ }
+
+ // once the simulated holder releases (try-with-resources above), the location opens normally
+ final TinkerStorageGraph graph = TinkerStorageGraph.open(config());
+ graph.close();
+ }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 4bdf5b8888..05e9970504 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
import java.io.File;
import java.io.RandomAccessFile;
+import java.nio.file.Files;
import java.util.Iterator;
import static org.junit.Assert.assertEquals;
@@ -112,14 +113,20 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.tx().commit();
graph.addVertex(T.id, 2, "value", 2);
graph.tx().commit();
- // do NOT close (avoid compaction) so the raw log is preserved
graph.tx().close();
- // simulate a crash mid-append by appending a partial (garbage) trailing frame to the log
+ // capture the raw log (two good frames) while the graph holds it, then close to release the directory lock.
+ // close() compacts, folding the log into a snapshot, so we reconstruct a "crashed" on-disk state below.
final File logFile = new File(location, GraphBinaryStorage.LOG_FILE);
+ final byte[] goodLog = Files.readAllBytes(logFile.toPath());
+ graph.close();
+
+ // recreate the pre-crash layout: no snapshot, a log of the two good frames plus a torn trailing frame (a
+ // length prefix promising 100 bytes with only a couple following), as an interrupted append would leave.
+ Files.deleteIfExists(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).toPath());
try (final RandomAccessFile raf = new RandomAccessFile(logFile, "rw")) {
- raf.seek(raf.length());
- // a length prefix promising 100 bytes, but only a couple follow — a torn write
+ raf.setLength(0);
+ raf.write(goodLog);
raf.writeInt(100);
raf.write(new byte[]{ 0x01, 0x02 });
}
(tinkerpop) 13/20: Encode TinkerStorageGraph elements as dictionary-referenced components
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 498bdb1c4001ae17064be1e0039cc4e984d9d41f
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 14:31:18 2026 +0000
Encode TinkerStorageGraph elements as dictionary-referenced components
Replace whole-DetachedVertex/DetachedEdge GraphBinary serialization with a
component codec: element ids and property values go through GraphBinary's
scalar serializers (a one-byte DataType tag + raw value), while labels,
property keys, and meta-property keys are dictionary-encoded to small integer
refs. New keys are emitted as OP_DICT_APPEND records within the same frame,
before their first use. Vertex-property ids are auto-generated and no longer
persisted (regenerated on load); element and edge ids are preserved.
Compaction preserves dictionary numbering and writes the whole dictionary as a
self-contained snapshot header, and dict-append decode is idempotent, so a log
surviving the compaction crash window still resolves its refs. This removes the
old whole-object codec (no dead code) and drops per-element key/label
repetition and the per-property envelope, shrinking the on-disk footprint.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 371 ++++++++++++++++++---
.../structure/storage/GraphBinaryStorageTest.java | 14 +-
.../storage/StorageCrashConsistencyTest.java | 15 +-
3 files changed, 334 insertions(+), 66 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index a5f3fd1c5c..1266dee705 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -19,34 +19,51 @@
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.structure.io.binary.DataType;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader;
import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter;
+import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializer;
import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
-import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* The GraphBinary {@link TinkerStorage} codec on top of {@link AbstractLogStorage}. The base owns the durable
* log-structured machinery (file layout, {@code VERSION} marker, CRC framing, replay fold, {@link SyncMode}
- * durability, crash-safe and threshold compaction); this class supplies only how an element is encoded and decoded,
- * using the GraphBinary serializers ({@link GraphBinaryWriter}/{@link GraphBinaryReader}).
+ * durability, crash-safe and threshold compaction); this class supplies only how an element is encoded and decoded.
* <p/>
- * Known limitation (write amplification): a commit records each changed element in full — a single property change on
- * a large element rewrites the whole element to the log. Elements are typically small and automatic compaction bounds
- * the resulting log growth, so this is accepted rather than mitigated with per-property deltas, which would complicate
- * the {@link TinkerStorageMutation} contract and the replay fold. The snapshot, by contrast, is streamed one element
- * at a time so compaction never holds a second full copy of the graph in heap.
+ * Rather than serialize a whole {@code DetachedVertex}/{@code DetachedEdge} (which repeats every property key and
+ * label as a full string on every element and wraps each property in a {@code VertexProperty} envelope), this codec
+ * writes element <em>components</em> directly: element ids and property values go through GraphBinary's scalar
+ * serializers (a one-byte {@link DataType} tag plus the raw value), while labels, property keys, and meta-property
+ * keys are dictionary-encoded to small integer refs. The dictionary is a single dense namespace built by first
+ * appearance; new entries are emitted as {@code OP_DICT_APPEND} records inside the same frame, before the entries
+ * that reference them, so a torn trailing frame drops a ref and its user atomically.
+ * <p/>
+ * Vertex-property ids are auto-generated and, by default, not persisted (they are regenerated on load); element and
+ * edge ids are always preserved. The snapshot streams one element per frame, so compaction never holds a second full
+ * copy of the graph in heap.
*/
public final class GraphBinaryStorage extends AbstractLogStorage {
@@ -54,71 +71,230 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
private static final byte OP_DEL_VERTEX = 2;
private static final byte OP_PUT_EDGE = 3;
private static final byte OP_DEL_EDGE = 4;
+ private static final byte OP_DICT_APPEND = 5;
- private final GraphBinaryWriter writer = new GraphBinaryWriter(TypeSerializerRegistry.INSTANCE);
- private final GraphBinaryReader reader = new GraphBinaryReader(TypeSerializerRegistry.INSTANCE);
+ private final TypeSerializerRegistry registry = TypeSerializerRegistry.INSTANCE;
+ private final GraphBinaryWriter writer = new GraphBinaryWriter(registry);
+ private final GraphBinaryReader reader = new GraphBinaryReader(registry);
/**
- * Serialize a commit record: txVersion, entry count, then each entry as an op byte followed by either the
- * serialized element (put) or the serialized id (delete).
+ * Shared string dictionary for labels, property keys, and meta-property keys. Dense ids assigned by first
+ * appearance; grows monotonically within a write session and is rebuilt fresh on compaction and on replay.
*/
+ private final Map<String, Integer> keyToId = new HashMap<>();
+ private final List<String> idToKey = new ArrayList<>();
+
+ @Override
+ protected void beginReplay() {
+ keyToId.clear();
+ idToKey.clear();
+ }
+
+ // --------------------------------------------------------------------------------------------- encode
+
@Override
protected byte[] encodeCommit(final long txVersion,
final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(txVersion);
- buffer.writeInt(changedVertices.size() + changedEdges.size());
+ final ByteBufferBuffer buf = new ByteBufferBuffer();
+ // register the strings introduced by this commit (deletes carry only an id, no strings)
+ final List<String> appends = new ArrayList<>();
+ for (final TinkerStorageMutation<TinkerVertex> m : changedVertices)
+ if (!m.isDeleted()) registerVertexStrings(m.element(), appends);
+ for (final TinkerStorageMutation<TinkerEdge> m : changedEdges)
+ if (!m.isDeleted()) registerEdgeStrings(m.element(), appends);
+
+ writeVarInt(buf, appends.size() + changedVertices.size() + changedEdges.size());
+ // dictionary appends first, so every ref below resolves during the fold
+ for (final String s : appends) {
+ buf.writeByte(OP_DICT_APPEND);
+ writeVarInt(buf, keyToId.get(s));
+ writeString(buf, s);
+ }
for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_VERTEX);
- writer.write(m.id(), buffer);
+ buf.writeByte(OP_DEL_VERTEX);
+ writeScalar(buf, m.id());
} else {
- buffer.writeByte(OP_PUT_VERTEX);
- // detach to a stable form independent of the transactional element
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ buf.writeByte(OP_PUT_VERTEX);
+ writeVertexRecord(buf, m.element());
}
}
for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
if (m.isDeleted()) {
- buffer.writeByte(OP_DEL_EDGE);
- writer.write(m.id(), buffer);
+ buf.writeByte(OP_DEL_EDGE);
+ writeScalar(buf, m.id());
} else {
- buffer.writeByte(OP_PUT_EDGE);
- writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ buf.writeByte(OP_PUT_EDGE);
+ writeEdgeRecord(buf, m.element());
+ }
+ }
+ return buf.toWrittenArray();
+ }
+
+ @Override
+ protected void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
+ // Preserve the existing dictionary numbering rather than renumbering: the compaction crash window can leave
+ // the new snapshot in place with the old log not yet truncated, and that log's refs use the current
+ // numbering. Register any not-yet-seen live strings (this only extends the dictionary, never renumbers), then
+ // emit the whole dictionary as a self-contained header frame so a snapshot-only replay resolves every ref.
+ final List<String> ignored = new ArrayList<>();
+ Iterator<Vertex> vertices = graph.vertices();
+ while (vertices.hasNext())
+ registerVertexStrings(vertices.next(), ignored);
+ Iterator<Edge> edges = graph.edges();
+ while (edges.hasNext())
+ registerEdgeStrings(edges.next(), ignored);
+
+ final ByteBufferBuffer dictBuf = new ByteBufferBuffer();
+ writeVarInt(dictBuf, idToKey.size());
+ for (int id = 0; id < idToKey.size(); id++) {
+ dictBuf.writeByte(OP_DICT_APPEND);
+ writeVarInt(dictBuf, id);
+ writeString(dictBuf, idToKey.get(id));
+ }
+ writeFrame(out, dictBuf.toWrittenArray());
+
+ // one element per frame; all keys are already in the dictionary header, so no per-frame appends
+ vertices = graph.vertices();
+ while (vertices.hasNext())
+ writeElementFrame(out, OP_PUT_VERTEX, vertices.next());
+ edges = graph.edges();
+ while (edges.hasNext())
+ writeElementFrame(out, OP_PUT_EDGE, edges.next());
+ }
+
+ /**
+ * Write one element as its own single-entry put frame. One element is held in memory at a time, so a large graph
+ * is never buffered whole.
+ */
+ private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
+ final ByteBufferBuffer buf = new ByteBufferBuffer();
+ writeVarInt(buf, 1);
+ buf.writeByte(op);
+ if (op == OP_PUT_VERTEX) writeVertexRecord(buf, (Vertex) element);
+ else writeEdgeRecord(buf, (Edge) element);
+ writeFrame(out, buf.toWrittenArray());
+ }
+
+ private void registerVertexStrings(final Vertex v, final List<String> appends) {
+ for (final String label : v.labels())
+ register(label, appends);
+ final Iterator<VertexProperty<Object>> vps = v.properties();
+ while (vps.hasNext()) {
+ final VertexProperty<Object> vp = vps.next();
+ register(vp.key(), appends);
+ final Iterator<Property<Object>> metas = vp.properties();
+ while (metas.hasNext())
+ register(metas.next().key(), appends);
+ }
+ }
+
+ private void registerEdgeStrings(final Edge e, final List<String> appends) {
+ register(e.label(), appends);
+ final Iterator<Property<Object>> props = e.properties();
+ while (props.hasNext())
+ register(props.next().key(), appends);
+ }
+
+ private void register(final String s, final List<String> appends) {
+ if (!keyToId.containsKey(s)) {
+ final int id = idToKey.size();
+ keyToId.put(s, id);
+ idToKey.add(s);
+ appends.add(s);
+ }
+ }
+
+ private void writeVertexRecord(final ByteBufferBuffer buf, final Vertex v) throws IOException {
+ writeScalar(buf, v.id());
+ final Set<String> labels = v.labels();
+ writeVarInt(buf, labels.size());
+ for (final String label : labels)
+ writeVarInt(buf, keyToId.get(label));
+
+ // group vertex properties by key so multi-properties (list/set) round-trip
+ final Map<String, List<VertexProperty<Object>>> groups = new LinkedHashMap<>();
+ final Iterator<VertexProperty<Object>> vps = v.properties();
+ while (vps.hasNext()) {
+ final VertexProperty<Object> vp = vps.next();
+ groups.computeIfAbsent(vp.key(), k -> new ArrayList<>()).add(vp);
+ }
+ writeVarInt(buf, groups.size());
+ for (final Map.Entry<String, List<VertexProperty<Object>>> group : groups.entrySet()) {
+ writeVarInt(buf, keyToId.get(group.getKey()));
+ final List<VertexProperty<Object>> values = group.getValue();
+ writeVarInt(buf, values.size());
+ for (final VertexProperty<Object> vp : values) {
+ writeScalar(buf, vp.value());
+ final List<Property<Object>> metas = new ArrayList<>();
+ vp.properties().forEachRemaining(metas::add);
+ writeVarInt(buf, metas.size());
+ for (final Property<Object> meta : metas) {
+ writeVarInt(buf, keyToId.get(meta.key()));
+ writeScalar(buf, meta.value());
+ }
}
}
- return buffer.toWrittenArray();
}
+ private void writeEdgeRecord(final ByteBufferBuffer buf, final Edge e) throws IOException {
+ writeScalar(buf, e.id());
+ writeVarInt(buf, keyToId.get(e.label()));
+ writeScalar(buf, e.outVertex().id());
+ writeScalar(buf, e.inVertex().id());
+ final List<Property<Object>> props = new ArrayList<>();
+ e.properties().forEachRemaining(props::add);
+ writeVarInt(buf, props.size());
+ for (final Property<Object> p : props) {
+ writeVarInt(buf, keyToId.get(p.key()));
+ writeScalar(buf, p.value());
+ }
+ }
+
+ // --------------------------------------------------------------------------------------------- decode
+
@Override
protected void decodeFrame(final byte[] record,
final Map<Object, DetachedVertex> vertices,
final Map<Object, DetachedEdge> edges) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
- buffer.readLong(); // txVersion, retained for diagnostics/future use
- final int entryCount = buffer.readInt();
+ final ByteBufferBuffer buf = new ByteBufferBuffer(record);
+ final int entryCount = readVarInt(buf);
for (int i = 0; i < entryCount; i++) {
- final byte op = buffer.readByte();
+ final byte op = buf.readByte();
switch (op) {
+ case OP_DICT_APPEND: {
+ final int id = readVarInt(buf);
+ final String s = readString(buf);
+ // idempotent: a snapshot header defines the whole dictionary, and a log surviving the compaction
+ // crash window may re-append entries the snapshot already established. Re-appending an existing
+ // id with the same string is a no-op; a mismatch or a gap is corruption.
+ if (id < idToKey.size()) {
+ if (!idToKey.get(id).equals(s))
+ throw new IOException(String.format("Corrupt storage: dictionary id %d redefined ('%s' vs '%s')", id, idToKey.get(id), s));
+ } else if (id == idToKey.size()) {
+ idToKey.add(s);
+ } else {
+ throw new IOException(String.format("Corrupt storage: dictionary append gap (got %d, expected <= %d)", id, idToKey.size()));
+ }
+ break;
+ }
case OP_PUT_VERTEX: {
- final Vertex v = reader.read(buffer);
- vertices.put(v.id(), (DetachedVertex) v);
+ final DetachedVertex v = readVertexRecord(buf);
+ vertices.put(v.id(), v);
break;
}
case OP_DEL_VERTEX: {
- final Object id = reader.read(buffer);
- vertices.remove(id);
+ vertices.remove(readScalar(buf));
break;
}
case OP_PUT_EDGE: {
- final Edge e = reader.read(buffer);
- edges.put(e.id(), (DetachedEdge) e);
+ final DetachedEdge e = readEdgeRecord(buf);
+ edges.put(e.id(), e);
break;
}
case OP_DEL_EDGE: {
- final Object id = reader.read(buffer);
- edges.remove(id);
+ edges.remove(readScalar(buf));
break;
}
default:
@@ -127,29 +303,114 @@ public final class GraphBinaryStorage extends AbstractLogStorage {
}
}
+ private DetachedVertex readVertexRecord(final ByteBufferBuffer buf) throws IOException {
+ final Object id = readScalar(buf);
+ final DetachedVertex.Builder b = DetachedVertex.build().setId(id);
+ final int labelCount = readVarInt(buf);
+ if (labelCount == 1) {
+ b.setLabel(idToKey.get(readVarInt(buf)));
+ } else if (labelCount > 1) {
+ final Set<String> labels = new LinkedHashSet<>();
+ for (int i = 0; i < labelCount; i++)
+ labels.add(idToKey.get(readVarInt(buf)));
+ b.setLabels(labels);
+ }
+ final int keyGroupCount = readVarInt(buf);
+ for (int g = 0; g < keyGroupCount; g++) {
+ final String key = idToKey.get(readVarInt(buf));
+ final int valueCount = readVarInt(buf);
+ for (int j = 0; j < valueCount; j++) {
+ final Object value = readScalar(buf);
+ final DetachedVertexProperty.Builder vpb = DetachedVertexProperty.build().setLabel(key).setValue(value);
+ final int metaCount = readVarInt(buf);
+ for (int m = 0; m < metaCount; m++) {
+ final String metaKey = idToKey.get(readVarInt(buf));
+ final Object metaValue = readScalar(buf);
+ vpb.addProperty(new DetachedProperty<>(metaKey, metaValue));
+ }
+ b.addProperty(vpb.create());
+ }
+ }
+ return b.create();
+ }
+
+ private DetachedEdge readEdgeRecord(final ByteBufferBuffer buf) throws IOException {
+ final Object id = readScalar(buf);
+ final String label = idToKey.get(readVarInt(buf));
+ final Object outVId = readScalar(buf);
+ final Object inVId = readScalar(buf);
+ final DetachedEdge.Builder b = DetachedEdge.build().setId(id).setLabel(label)
+ .setOutV(DetachedVertex.build().setId(outVId).create())
+ .setInV(DetachedVertex.build().setId(inVId).create());
+ final int propCount = readVarInt(buf);
+ for (int i = 0; i < propCount; i++) {
+ final String key = idToKey.get(readVarInt(buf));
+ final Object value = readScalar(buf);
+ b.addProperty(new DetachedProperty<>(key, value));
+ }
+ return b.create();
+ }
+
+ // --------------------------------------------------------------------------------------------- primitives
+
/**
- * Stream the current committed state as one framed put record per vertex and per edge, so peak memory is bounded
- * to a single element rather than materializing the whole graph as one byte array.
+ * Write a value as a one-byte {@link DataType} tag followed by the raw value (no value-flag byte). A {@code null}
+ * is a single {@link DataType#UNSPECIFIED_NULL} tag.
*/
- @Override
- protected void writeSnapshot(final AbstractTinkerGraph graph, final DataOutputStream out) throws IOException {
- final Iterator<Vertex> vertexIterator = graph.vertices();
- while (vertexIterator.hasNext())
- writeElementFrame(out, OP_PUT_VERTEX, vertexIterator.next());
- final Iterator<Edge> edgeIterator = graph.edges();
- while (edgeIterator.hasNext())
- writeElementFrame(out, OP_PUT_EDGE, edgeIterator.next());
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private void writeScalar(final ByteBufferBuffer buf, final Object value) throws IOException {
+ if (value == null) {
+ buf.writeByte(DataType.UNSPECIFIED_NULL.getCodeByte());
+ return;
+ }
+ final TypeSerializer serializer = registry.getSerializer(value.getClass());
+ buf.writeByte(serializer.getDataType().getCodeByte());
+ serializer.writeValue(value, buf, writer, false);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ private Object readScalar(final ByteBufferBuffer buf) throws IOException {
+ final DataType dataType = DataType.get(Byte.toUnsignedInt(buf.readByte()));
+ if (dataType == DataType.UNSPECIFIED_NULL)
+ return null;
+ final TypeSerializer serializer = registry.getSerializer(dataType);
+ return serializer.readValue(buf, reader, false);
+ }
+
+ private static void writeString(final ByteBufferBuffer buf, final String s) {
+ final byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
+ writeVarInt(buf, bytes.length);
+ buf.writeBytes(bytes);
+ }
+
+ private static String readString(final ByteBufferBuffer buf) {
+ final byte[] bytes = new byte[readVarInt(buf)];
+ buf.readBytes(bytes);
+ return new String(bytes, StandardCharsets.UTF_8);
}
/**
- * Encode a single element as a one-entry put record and write it as a framed record to {@code out}.
+ * Unsigned LEB128 varint. Counts and dictionary refs are small and non-negative, so they cost one byte in the
+ * common case.
*/
- private void writeElementFrame(final DataOutputStream out, final byte op, final Object element) throws IOException {
- final ByteBufferBuffer buffer = new ByteBufferBuffer();
- buffer.writeLong(0L); // snapshot records have no single tx version
- buffer.writeInt(1);
- buffer.writeByte(op);
- writer.write(DetachedFactory.detach(element, true), buffer);
- writeFrame(out, buffer.toWrittenArray());
+ private static void writeVarInt(final ByteBufferBuffer buf, final int value) {
+ int v = value;
+ while ((v & ~0x7F) != 0) {
+ buf.writeByte((v & 0x7F) | 0x80);
+ v >>>= 7;
+ }
+ buf.writeByte(v & 0x7F);
+ }
+
+ private static int readVarInt(final ByteBufferBuffer buf) {
+ int result = 0;
+ int shift = 0;
+ byte b;
+ do {
+ b = buf.readByte();
+ result |= (b & 0x7F) << shift;
+ shift += 7;
+ } while ((b & 0x80) != 0);
+ return result;
}
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index c113511176..cbde701a22 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -127,10 +127,10 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.tx().commit();
graph.compact();
- // the snapshot must be written as one framed record per element (3 vertices + 2 edges = 5), rather than a
- // single whole-graph frame, so compaction never buffers the entire graph in one array
+ // the snapshot must be streamed: a dictionary header frame plus one framed record per element (3 vertices +
+ // 2 edges = 5), rather than a single whole-graph frame, so compaction never buffers the entire graph at once
final File snapshotFile = new File(location, GraphBinaryStorage.SNAPSHOT_FILE);
- assertEquals(5, countFrames(snapshotFile));
+ assertEquals(1 + 5, countFrames(snapshotFile));
graph.close();
// and the streamed snapshot must reopen to exactly the same graph
@@ -145,9 +145,9 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
@Test
public void shouldStreamSnapshotFrameByFrameAtScale() {
// Bounded-memory proxy for the streaming snapshot path: rather than measure heap (flaky, JVM-dependent), assert
- // the observable streaming property holds at scale — a large graph is written as exactly one frame per element,
- // never one whole-graph frame — and round-trips intact. This is the property that keeps compaction from
- // materializing a second full copy of the graph in memory; it is not a hard OOM assertion.
+ // the observable streaming property holds at scale — a large graph is written as a dictionary header frame plus
+ // one frame per element, never one whole-graph frame — and round-trips intact. This is the property that keeps
+ // compaction from materializing a second full copy of the graph in memory; it is not a hard OOM assertion.
final int vertexCount = 500;
final int edgeCount = 499;
final TinkerStorageGraph graph = open();
@@ -161,7 +161,7 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.compact();
try {
- assertEquals(vertexCount + edgeCount, countFrames(new File(location, GraphBinaryStorage.SNAPSHOT_FILE)));
+ assertEquals(1 + vertexCount + edgeCount, countFrames(new File(location, GraphBinaryStorage.SNAPSHOT_FILE)));
} catch (Exception ex) {
throw new RuntimeException(ex);
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
index 614cfdb0ba..8f3c475f13 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
@@ -123,10 +123,17 @@ public class StorageCrashConsistencyTest {
@Test
public void shouldRecoverDurableCommitWithNoSnapshot() throws Exception {
- // WAL guarantee: a commit whose frame was durably written to the log, with no compaction having run, must
- // recover on reopen even though no snapshot exists.
- captureBuildingBlocks();
- Files.write(logFile.toPath(), logV2);
+ // WAL guarantee: a commit durably written to the log with no compaction (no snapshot) recovers on reopen. On
+ // a fresh store the dictionary starts empty, so the commit frame is self-contained (it carries its own
+ // dictionary appends), which is exactly the real "log only, never compacted" case.
+ final TinkerStorageGraph g = open();
+ g.addVertex(T.id, 2, "value", 2);
+ g.tx().commit();
+ g.tx().close();
+ final byte[] selfContainedLog = Files.readAllBytes(logFile.toPath());
+ g.close(); // compaction on close would fold the log away; the raw log was captured above
+ resetFiles();
+ Files.write(logFile.toPath(), selfContainedLog);
assertReopensTo(2);
}
(tinkerpop) 11/20: Document TinkerStorageGraph durability, compaction, and locking
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit adcd29a6af3effc061ec6c748837b9815a157d8e
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 23:53:23 2026 +0000
Document TinkerStorageGraph durability, compaction, and locking
Bring the TinkerGraph reference persistence section up to date with the storage
engine's settings: the gremlin.tinkergraph.storage.sync durability modes
(commit/os) and gremlin.tinkergraph.storage.compactThreshold auto-compaction
knob are added to the configuration table, and the persistence prose now covers
commit durability, log compaction, single-writer directory locking, and the
storage-format version check with g.io() export as the migration path.
Assisted-by: Claude Code:claude-opus-4-8
---
.../reference/implementations-tinkergraph.asciidoc | 26 ++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index 6f7c417bfb..b657d0c9f4 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -178,6 +178,14 @@ to disk. The value is either a built-in engine name (`graphbinary`) or a fully q
only valid on `TinkerStorageGraph` and is ignored by the in-memory `TinkerMemoryGraph`.
|gremlin.tinkergraph.graphLocation |The directory in which `TinkerStorageGraph` stores its durable data. Required when
`gremlin.tinkergraph.storage` is set and ignored otherwise.
+|gremlin.tinkergraph.storage.sync |The durability applied to each committed transaction by a `TinkerStorageGraph`
+storage engine. `commit` (default) forces each commit to disk so an acknowledged commit survives an operating system
+crash or power loss. `os` only flushes to the operating system, so a commit survives a crash of the JVM process but
+may be lost on an operating system crash or power loss. Only meaningful when `gremlin.tinkergraph.storage` is set.
+|gremlin.tinkergraph.storage.compactThreshold |The size in bytes at which a `TinkerStorageGraph` storage engine
+automatically compacts its append log, bounding the log growth and restart time of a long-running graph that is never
+explicitly closed. Defaults to `67108864` (64 MB). A value of `0` disables automatic compaction, leaving it to
+`close()` or an explicit `compact()`. Only meaningful when `gremlin.tinkergraph.storage` is set.
|=========================================================
NOTE: To use <<tinkergraph-gremlin-tx, transactions>> and <<tinkergraph-gremlin-persistence, persistence>>, configure
@@ -437,6 +445,24 @@ The graph remains the authoritative in-memory copy and is mirrored to disk, so a
memory. The write to the storage log happens before the in-memory commit is applied, so a failure to persist aborts the
transaction and leaves the on-disk data and the in-memory graph consistent.
+The durability of each commit is governed by the `gremlin.tinkergraph.storage.sync` setting. Under the default
+`commit` mode every committed transaction is forced to disk, so an acknowledged commit survives an operating system
+crash or power loss. The `os` mode instead flushes only to the operating system, which is faster but leaves a commit
+recoverable only across a crash of the JVM process, not an operating system crash or power loss.
+
+A storage engine that appends each commit to a log reclaims space by compacting that log into a fresh snapshot of the
+committed state. Compaction runs when the graph is closed and can be requested explicitly through
+`TinkerStorageGraph.compact()`. So that a long-running graph which is never explicitly closed does not accumulate an
+unbounded log, compaction also runs automatically once the log grows past `gremlin.tinkergraph.storage.compactThreshold`
+bytes. Setting that threshold to `0` disables automatic compaction.
+
+A storage location may be opened by only one graph at a time. `TinkerStorageGraph` takes an exclusive lock on the
+storage directory when it opens, so a second attempt to open the same location, whether from the same JVM or another
+process, fails rather than corrupting the data. The lock is released when the graph is closed. A store also records
+the storage format version it was written with. Opening a store written in a format this version cannot read fails
+with a clear error rather than misreading the data. There is no in-place format migration. To move a graph across an
+incompatible storage format, export it with the `io()` step before upgrading and read it back afterward.
+
Persistence is distinct from interchange. `TinkerMemoryGraph` is purely in-memory and does not persist. To move data in
or out of any TinkerGraph in an interchange format such as GraphML, GraphSON, or Gryo, use the `io()` step directly:
(tinkerpop) 16/20: Broaden TinkerStorage TCK: value types, ids, varints, unicode round-trips
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit bc79b74142a29412b9932083b3c8e4d3241733d6
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 16:24:13 2026 +0000
Broaden TinkerStorage TCK: value types, ids, varints, unicode round-trips
Add conformance round-trip coverage for the storage codec's breadth: a
value-type matrix (int/long/float/double/boolean/byte/short/char/string/UUID/
BigInteger/BigDecimal/OffsetDateTime/Duration), collection-valued properties
(List/Map/Set), null values, heterogeneous same-key types, non-Long (UUID and
String) element ids, unicode keys and values, and a large-schema graph whose
>127 distinct keys and >127 values under one key exercise the multi-byte varint
path that small graphs never reach.
Assisted-by: Claude Code:claude-opus-4-8
---
.../AbstractTinkerStorageConformanceTest.java | 206 +++++++++++++++++++++
1 file changed, 206 insertions(+)
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
index 552cd37da4..f5ec5033f4 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
@@ -32,15 +32,26 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.time.Duration;
+import java.time.OffsetDateTime;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
@@ -284,6 +295,201 @@ public abstract class AbstractTinkerStorageConformanceTest {
}
}
+ @Test
+ public void shouldRoundTripDiverseValueTypes() {
+ final Map<String, Object> values = new LinkedHashMap<>();
+ values.put("int", 42);
+ values.put("long", 42L);
+ values.put("float", 1.5f);
+ values.put("double", 2.5d);
+ values.put("bool", true);
+ values.put("byte", (byte) 7);
+ values.put("short", (short) 9);
+ values.put("char", 'x');
+ values.put("string", "hello");
+ values.put("uuid", new UUID(12L, 34L));
+ values.put("bigint", new BigInteger("123456789012345678901234567890"));
+ values.put("bigdec", new BigDecimal("3.14159265358979"));
+ values.put("datetime", OffsetDateTime.parse("2020-01-02T03:04:05Z"));
+ values.put("duration", Duration.ofSeconds(90));
+
+ TinkerStorageGraph graph = open();
+ try {
+ final Vertex v = graph.addVertex(T.id, 1);
+ values.forEach(v::property);
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ final Vertex v = graph.vertices(1).next();
+ values.forEach((k, expected) -> assertEquals(k, expected, v.value(k)));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripCollectionValuedProperties() {
+ final List<Object> list = Arrays.asList(1, "two", 3.0d);
+ final Map<String, Object> map = new LinkedHashMap<>();
+ map.put("a", 1);
+ map.put("b", "two");
+ final Set<Object> set = new LinkedHashSet<>(Arrays.asList("x", "y", "z"));
+
+ TinkerStorageGraph graph = open();
+ try {
+ final Vertex v = graph.addVertex(T.id, 1);
+ v.property("list", list);
+ v.property("map", map);
+ v.property("set", set);
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ final Vertex v = graph.vertices(1).next();
+ assertEquals(list, v.value("list"));
+ assertEquals(map, v.value("map"));
+ assertEquals(set, v.value("set"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripNullPropertyValue() {
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES, true);
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ try {
+ final Vertex v = graph.addVertex(T.id, 1);
+ v.property("maybe", null);
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = TinkerStorageGraph.open(conf);
+ try {
+ final VertexProperty<Object> vp = graph.vertices(1).next().<Object>properties("maybe").next();
+ assertTrue(vp.isPresent());
+ assertNull(vp.value());
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripHeterogeneousSameKeyTypes() {
+ TinkerStorageGraph graph = open();
+ try {
+ graph.addVertex(T.id, 1, "k", 42); // Integer
+ graph.addVertex(T.id, 2, "k", 42L); // Long
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ assertEquals(Integer.valueOf(42), graph.vertices(1).next().value("k"));
+ assertEquals(Long.valueOf(42L), graph.vertices(2).next().value("k"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripUuidElementIds() {
+ roundTripElementIds(new UUID(0L, 1L), new UUID(0L, 2L), new UUID(0L, 10L));
+ }
+
+ @Test
+ public void shouldRoundTripStringElementIds() {
+ roundTripElementIds("v-1", "v-2", "e-10");
+ }
+
+ // uses the default ANY id manager so any id type is accepted verbatim; the point is that the storage codec
+ // round-trips non-Long element ids through its scalar id encoding
+ private void roundTripElementIds(final Object outId, final Object inId, final Object edgeId) {
+ TinkerStorageGraph graph = open();
+ try {
+ final Vertex a = graph.addVertex(T.id, outId, "name", "a");
+ final Vertex b = graph.addVertex(T.id, inId, "name", "b");
+ a.addEdge("knows", b, T.id, edgeId, "weight", 0.5d);
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ assertEquals("a", graph.vertices(outId).next().value("name"));
+ assertEquals(outId, graph.vertices(outId).next().id());
+ final Edge e = graph.edges(edgeId).next();
+ assertEquals("knows", e.label());
+ assertEquals(outId, e.outVertex().id());
+ assertEquals(inId, e.inVertex().id());
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripLargeSchemaAcrossVarintBoundary() {
+ // >127 distinct keys and >127 values under one key push dictionary refs and counts past the single-byte
+ // LEB128 range, exercising the multi-byte varint path that small graphs never reach
+ final int n = 200;
+ // list cardinality so the >127 values under "multi" survive reopen (reconstruction takes cardinality from
+ // graph config, not the stored record)
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, "list");
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ try {
+ final Vertex v = graph.addVertex(T.id, 1);
+ for (int i = 0; i < n; i++)
+ v.property("key" + i, i);
+ for (int i = 0; i < n; i++)
+ v.property(VertexProperty.Cardinality.list, "multi", i);
+ graph.tx().commit();
+ graph.compact(); // also exercises a dictionary header with >127 entries
+ } finally {
+ graph.close();
+ }
+ graph = TinkerStorageGraph.open(conf);
+ try {
+ final Vertex v = graph.vertices(1).next();
+ for (int i = 0; i < n; i++)
+ assertEquals("key" + i, Integer.valueOf(i), v.value("key" + i));
+ assertEquals(n, countOf(v.properties("multi")));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripUnicodeKeysAndValues() {
+ TinkerStorageGraph graph = open();
+ try {
+ final Vertex v = graph.addVertex(T.id, 1);
+ v.property("naïve", "café");
+ v.property("日本語", "テスト");
+ v.property("emoji", "party🎉");
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ final Vertex v = graph.vertices(1).next();
+ assertEquals("café", v.value("naïve"));
+ assertEquals("テスト", v.value("日本語"));
+ assertEquals("party🎉", v.value("emoji"));
+ } finally {
+ graph.close();
+ }
+ }
+
private static long countOf(final Iterator<?> it) {
long count = 0;
while (it.hasNext()) {
(tinkerpop) 02/20: Move TinkerStorageGraph storage changelog entries to 4.0.0 section
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 93702c87a89552ec122452577a743f29f2781f88
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Thu Jul 30 14:48:57 2026 +0000
Move TinkerStorageGraph storage changelog entries to 4.0.0 section
The pluggable storage and TinkerMemoryGraph persistence-removal entries
were landing in the released beta.3 section; move them up beside the
related TinkerGraph refactoring entries in the unreleased 4.0.0 section.
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 78dff7595a..0f7480173a 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,6 +28,8 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message.
* Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)*
* Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)*
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine.
+* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
[[release-4-0-0-beta-3]]
=== TinkerPop 4.0.0-beta.3 (July 20, 2026)
@@ -56,8 +58,6 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-dotnet` deflate response decompression, which threw on the server's zlib-framed output because it used `DeflateStream` (raw DEFLATE, RFC 1951) instead of `ZLibStream` (zlib, RFC 1950); the bug was previously masked because compression was off by default.
* Fixed `gremlin-dotnet` SSL options cloning (used on the skip-certificate-validation path) to copy `ClientCertificateContext` and `AllowTlsResume`, which were previously dropped, breaking mTLS client certificates and silently re-enabling TLS resumption.
* Fixed `gremlin-python` read timeout to derive from a single source (aiohttp `sock_read`), removing a redundant `async_timeout` read wrapper that could race it; a read timeout now deterministically raises `ReadTimeoutError` (a builtin `TimeoutError` subclass).
-* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine.
-* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
* Removed `Transaction.open()` in favor of `begin()`, which is now the single transaction-start primitive across embedded and remote contexts.
* Changed `begin()` and `close()` to be idempotent and calling it when a transaction is already in that state no longer throws.
* Added `maxTransactionLifetimeMillis` setting to Gremlin Server, an absolute cap on the total age of an HTTP transaction that interrupts a running operation and rolls the transaction back when it fires (default 600000ms, set to `0` to disable).
(tinkerpop) 01/20: Add pluggable storage layer to TinkerStorageGraph
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit e08c6f823a88e11dbb58e8dc89079a1f4b1e4587
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Jul 14 21:44:15 2026 +0000
Add pluggable storage layer to TinkerStorageGraph
TinkerStorageGraph can now durably persist committed transactions to disk
through a pluggable storage engine, selected with the new
gremlin.tinkergraph.storage config key with graphLocation as the storage
directory. A GraphBinary write-ahead-log engine ships as the reference
implementation; custom engines implement the TinkerStorage SPI.
TinkerMemoryGraph is now purely in-memory: the old load-on-open /
save-on-close behavior and the graphFormat key are removed. Use
TinkerStorageGraph for durability or g.io() for interchange.
SimpleAuthenticator now reads its credential store explicitly since the
in-memory graph no longer auto-loads.
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 2 +
.../reference/implementations-tinkergraph.asciidoc | 108 +++---
docs/src/upgrade/release-4.x.x.asciidoc | 50 +++
gremlin-console/conf/tinkergraph-gryo.properties | 6 +-
.../gremlin/server/auth/SimpleAuthenticator.java | 47 +++
.../tinkergraph/structure/AbstractTinkerGraph.java | 100 +++---
.../gremlin/tinkergraph/structure/TinkerGraph.java | 14 +-
.../tinkergraph/structure/TinkerMemoryGraph.java | 20 +-
.../tinkergraph/structure/TinkerStorageGraph.java | 36 +-
.../tinkergraph/structure/TinkerTransaction.java | 39 +++
.../structure/storage/ByteBufferBuffer.java | 336 ++++++++++++++++++
.../structure/storage/DefaultStorage.java | 40 +++
.../structure/storage/GraphBinaryStorage.java | 379 +++++++++++++++++++++
.../structure/storage/TinkerStorage.java | 100 ++++++
.../structure/storage/TinkerStorageMutation.java | 67 ++++
.../tinkergraph/TinkerMemoryGraphProvider.java | 20 --
.../tinkergraph/TinkerMemoryGraphUUIDProvider.java | 6 -
.../tinkergraph/TinkerStorageGraphProvider.java | 23 +-
.../structure/TinkerMemoryGraphTest.java | 180 ----------
.../AbstractTinkerStorageConformanceTest.java | 242 +++++++++++++
.../structure/storage/GraphBinaryStorageTest.java | 92 +++++
21 files changed, 1577 insertions(+), 330 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index e71354681a..78dff7595a 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -56,6 +56,8 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-dotnet` deflate response decompression, which threw on the server's zlib-framed output because it used `DeflateStream` (raw DEFLATE, RFC 1951) instead of `ZLibStream` (zlib, RFC 1950); the bug was previously masked because compression was off by default.
* Fixed `gremlin-dotnet` SSL options cloning (used on the skip-certificate-validation path) to copy `ClientCertificateContext` and `AllowTlsResume`, which were previously dropped, breaking mTLS client certificates and silently re-enabling TLS resumption.
* Fixed `gremlin-python` read timeout to derive from a single source (aiohttp `sock_read`), removing a redundant `async_timeout` read wrapper that could race it; a read timeout now deterministically raises `ReadTimeoutError` (a builtin `TimeoutError` subclass).
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine.
+* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
* Removed `Transaction.open()` in favor of `begin()`, which is now the single transaction-start primitive across embedded and remote contexts.
* Changed `begin()` and `close()` to be idempotent and calling it when a transaction is already in that state no longer throws.
* Added `maxTransactionLifetimeMillis` setting to Gremlin Server, an absolute cap on the total age of an HTTP transaction that interrupts a running operation and rolls the transaction back when it fires (default 600000ms, set to `0` to disable).
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index 4e3c873e3c..6f7c417bfb 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -39,13 +39,13 @@ under the License.
</dependency>
----
-image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory (with optional
-persistence), graph engine that provides both OLTP and OLAP functionality. It is non-transactional by default but does
+image:tinkerpop-character.png[width=100,float=left] TinkerGraph is a single machine, in-memory graph engine that
+provides both OLTP and OLAP functionality. It is non-transactional by default but does
have a lightweight transactional form that can be instantiated offering simple `ThreadLocal` transactions supporting
-`read committed` transaction isolation. As of 4.0.0, `TinkerGraph` is an interface with two implementations:
-`TinkerMemoryGraph`, the in-memory, non-transactional implementation that `TinkerGraph.open()` constructs, and
-`TinkerStorageGraph`, the transactional implementation formerly named `TinkerTransactionGraph`. TinkerGraph is
-deployed with TinkerPop and serves as the reference
+`read committed` transaction isolation, with optional durable persistence to disk. As of 4.0.0, `TinkerGraph` is an
+interface with two implementations: `TinkerMemoryGraph`, the in-memory, non-transactional implementation that
+`TinkerGraph.open()` constructs, and `TinkerStorageGraph`, the transactional implementation formerly named
+`TinkerTransactionGraph`. TinkerGraph is deployed with TinkerPop and serves as the reference
implementation for other providers to study in order to understand the semantics of the various methods of the
TinkerPop API. Its status as a reference implementation does not however imply that it is not suitable for production.
TinkerGraph has many practical use cases in production applications and their development. Some examples of TinkerGraph
@@ -172,19 +172,16 @@ TinkerGraph has several settings that can be provided on creation via `Configura
|gremlin.tinkergraph.defaultVertexPropertyCardinality |The default `VertexProperty.Cardinality` to use when `Vertex.property(k,v)` is called.
|gremlin.tinkergraph.allowNullPropertyValues |A boolean value that determines whether or not `null` property values are allowed and defaults to `false`.
|gremlin.tinkergraph.vertexLabelCardinality |The `LabelCardinality` for vertices. Options are `ONE` (default, single immutable label, backward-compatible with 3.x), `ONE_OR_MORE` (at least one label, mutable), or `ZERO_OR_MORE` (zero or more labels). See <<tinkergraph-multi-label>>.
-|gremlin.tinkergraph.graphLocation |The path and file name for where TinkerGraph should persist the graph data. If a
-value is specified here, the `gremlin.tinkergraph.graphFormat` should also be specified. If this value is not
-included (default), then the graph will stay in-memory and not be loaded/persisted to disk.
-|gremlin.tinkergraph.graphFormat |The format to use to serialize the graph which may be one of the following:
-`graphml`, `graphson`, `gryo`, or a fully qualified class name that implements Io.Builder interface (which allows for
-external third party graph reader/writer formats to be used for persistence).
-If a value is specified here, then the `gremlin.tinkergraph.graphLocation` should
-also be specified. If this value is not included (default), then the graph will stay in-memory and not be
-loaded/persisted to disk.
+|gremlin.tinkergraph.storage |The durable storage engine used by `TinkerStorageGraph` to persist committed transactions
+to disk. The value is either a built-in engine name (`graphbinary`) or a fully qualified class name of a
+`TinkerStorage` implementation. When not specified (default), the graph holds data only in memory. This setting is
+only valid on `TinkerStorageGraph` and is ignored by the in-memory `TinkerMemoryGraph`.
+|gremlin.tinkergraph.graphLocation |The directory in which `TinkerStorageGraph` stores its durable data. Required when
+`gremlin.tinkergraph.storage` is set and ignored otherwise.
|=========================================================
-NOTE: To use <<tinkergraph-gremlin-tx, transactions>>, configure `gremlin.graph` as
-`org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph`.
+NOTE: To use <<tinkergraph-gremlin-tx, transactions>> and <<tinkergraph-gremlin-persistence, persistence>>, configure
+`gremlin.graph` as `org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph`.
The `IdManager` settings above refer to how TinkerGraph will control identifiers for vertices, edges and vertex
properties. There are several options for each of these settings: `ANY`, `LONG`, `INTEGER`, `UUID`, `STRING` or the
@@ -197,32 +194,10 @@ type as well as generate new identifiers with that specified type.
TIP: Setting the `IdManager` to `ANY` also allows `String` type ID values to be used.
-If the TinkerGraph is configured for persistence with `gremlin.tinkergraph.graphLocation` and
-`gremlin.tinkergraph.graphFormat`, then the graph will be written to the specified location with the specified
-format when `Graph.close()` is called. In addition, if these settings are present, TinkerGraph will attempt to
-load the graph from the specified location.
-
-IMPORTANT: If choosing `graphson` as the `gremlin.tinkergraph.graphFormat` with untyped GraphSON, be sure to also
-establish the various `IdManager` settings to ensure that identifiers are properly coerced to the appropriate types,
-as untyped GraphSON can lose the identifier's type during serialization (i.e. it will assume `Integer` when the
-default for TinkerGraph is `Long`, which could lead to load errors that result in a message like, "Vertex with id
-already exists"). Typed GraphSON preserves the identifier's type across the serialization round-trip (for example, a
-`Long` identifier is written as `{"@type":"g:Int64","@value":...}`), so the `IdManager` coercion above is not
-required in that case.
-
-These settings can be supplied programmatically by building a `Configuration` and passing it to `TinkerGraph.open()`.
-For example, to persist to disk as GraphSON while keeping `Long` identifiers:
-
-[source,java]
-----
-BaseConfiguration conf = new BaseConfiguration();
-conf.setProperty("gremlin.tinkergraph.graphLocation", "/tmp/tinkergraph.json");
-conf.setProperty("gremlin.tinkergraph.graphFormat", "graphson");
-conf.setProperty("gremlin.tinkergraph.vertexIdManager", "LONG");
-Graph graph = TinkerGraph.open(conf);
-// ... work with the graph ...
-graph.close(); // persists the graph to the configured graphLocation
-----
+Durable persistence is provided by `TinkerStorageGraph` through its pluggable storage layer and is described in the
+<<tinkergraph-gremlin-persistence, persistence section>>. The in-memory `TinkerMemoryGraph` holds no data across JVM
+restarts. Moving data in and out of any TinkerGraph in an interchange format is handled on demand by the `io()` step
+rather than by graph configuration, as shown below.
It is important to consider the data being imported to TinkerGraph with respect to `defaultVertexPropertyCardinality`
setting. For example, if a `.gryo` file is known to contain multi-property data, be sure to set the default
@@ -424,6 +399,53 @@ g.V().valueMap()
<4> Add a second vertex without committing
<5> Rollback the change
+[[tinkergraph-gremlin-persistence]]
+=== Persistence
+
+`TinkerStorageGraph` can durably persist to disk through a pluggable storage layer built on its transaction support.
+When a storage engine is configured, the changeset of each committed transaction is written to disk, and the graph is
+rebuilt from that data when it is opened again. The in-memory `TinkerMemoryGraph` does not retain data across restarts.
+
+A storage engine is selected with the `gremlin.tinkergraph.storage` configuration key, and
+`gremlin.tinkergraph.graphLocation` names the directory that holds the durable data. The value of the storage key is
+either a built-in engine name or the fully qualified class name of a `TinkerStorage` implementation, following the same
+enum-name-or-class-name convention as the `IdManager` settings. The built-in `graphbinary` engine records committed
+transactions as an append-only log serialized with GraphBinary and folds that log into a compact snapshot when the
+graph is closed.
+
+[source,groovy]
+----
+conf = new BaseConfiguration()
+conf.setProperty("gremlin.graph", "org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph")
+conf.setProperty("gremlin.tinkergraph.storage", "graphbinary")
+conf.setProperty("gremlin.tinkergraph.graphLocation", "/data/mygraph")
+
+graph = TinkerStorageGraph.open(conf)
+g = traversal().with(graph)
+g.addV("person").property("name","marko").iterate()
+g.tx().commit()
+graph.close()
+
+// reopening the same location restores the committed data
+graph = TinkerStorageGraph.open(conf)
+g = traversal().with(graph)
+g.V().count()
+==>1
+----
+
+The graph remains the authoritative in-memory copy and is mirrored to disk, so a persisted graph must still fit in
+memory. The write to the storage log happens before the in-memory commit is applied, so a failure to persist aborts the
+transaction and leaves the on-disk data and the in-memory graph consistent.
+
+Persistence is distinct from interchange. `TinkerMemoryGraph` is purely in-memory and does not persist. To move data in
+or out of any TinkerGraph in an interchange format such as GraphML, GraphSON, or Gryo, use the `io()` step directly:
+
+[source,groovy]
+----
+g.io("/tmp/graph.kryo").write().iterate()
+g.io("/tmp/graph.kryo").read().iterate()
+----
+
[[tinkergraph-gql]]
=== TinkerGQL
diff --git a/docs/src/upgrade/release-4.x.x.asciidoc b/docs/src/upgrade/release-4.x.x.asciidoc
index 36a01282cc..1097f9b075 100644
--- a/docs/src/upgrade/release-4.x.x.asciidoc
+++ b/docs/src/upgrade/release-4.x.x.asciidoc
@@ -71,6 +71,56 @@ which affects providers that extended them.
See: link:https://lists.apache.org/thread/2zt62kvfssh6xz5vnf2lk1g7cstq9vod[DISCUSS thread]
+==== TinkerStorageGraph Pluggable Disk Storage
+
+`TinkerStorageGraph` gained the optional disk storage anticipated by its rename. A storage engine is selected with the
+new `gremlin.tinkergraph.storage` configuration key, and `gremlin.tinkergraph.graphLocation` names the directory that
+holds the durable data. When a storage engine is configured, each committed transaction is durably written to disk and
+the graph is rebuilt from that data when it is opened again, so a graph survives a restart of the JVM.
+
+The reference engine, `graphbinary`, records committed transactions as an append-only log serialized with GraphBinary
+and folds that log into a compact snapshot on close. The storage layer is pluggable: the value of the storage key may
+also be the fully-qualified class name of a custom engine, following the same convention as the `IdManager` selection.
+
+[source,groovy]
+----
+conf = new BaseConfiguration()
+conf.setProperty('gremlin.graph', 'org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph')
+conf.setProperty('gremlin.tinkergraph.storage', 'graphbinary')
+conf.setProperty('gremlin.tinkergraph.graphLocation', '/data/mygraph')
+
+graph = TinkerStorageGraph.open(conf)
+g = traversal().with(graph)
+g.addV('person').property('name','marko').iterate()
+g.tx().commit()
+graph.close()
+
+// reopening the same location restores the committed data
+graph = TinkerStorageGraph.open(conf)
+g = traversal().with(graph)
+g.V().count().next()
+==>1
+----
+
+The in-memory `TinkerMemoryGraph` no longer persists to disk. Earlier versions of TinkerGraph would automatically read
+from `gremlin.tinkergraph.graphLocation` on open and write back to it on close, using the `gremlin.tinkergraph.graphFormat`
+interchange format. That automatic behavior is removed, and `TinkerMemoryGraph` now ignores both keys and reports
+`FEATURE_PERSISTENCE` as `false`. Durable persistence is the responsibility of `TinkerStorageGraph` and its storage
+engine, while moving data in and out of any graph in an interchange format remains the job of the `io()` step:
+
+[source,groovy]
+----
+// interchange, on demand, for any graph
+g.io('/tmp/graph.kryo').write().iterate()
+g.io('/tmp/graph.kryo').read().iterate()
+----
+
+Configurations that previously relied on the in-memory graph loading itself from `graphLocation` on open must either
+call `io().read()` explicitly or switch to `TinkerStorageGraph` with a storage engine. The `gremlin.tinkergraph.graphFormat`
+key is retired.
+
+See: <<tinkergraph-gremlin,TinkerGraph>>
+
=== Upgrading for Providers
==== Graph System Providers
diff --git a/gremlin-console/conf/tinkergraph-gryo.properties b/gremlin-console/conf/tinkergraph-gryo.properties
index 4c2684237c..312dc3f9b6 100644
--- a/gremlin-console/conf/tinkergraph-gryo.properties
+++ b/gremlin-console/conf/tinkergraph-gryo.properties
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph
-gremlin.tinkergraph.graphFormat=gryo
-gremlin.tinkergraph.graphLocation=/tmp/tinkergraph.kryo
+gremlin.tinkergraph.storage=graphbinary
+gremlin.tinkergraph.graphLocation=/tmp/tinkergraph
diff --git a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java
index afdae0d2de..435316cd08 100644
--- a/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java
+++ b/gremlin-server/src/main/java/org/apache/tinkerpop/gremlin/server/auth/SimpleAuthenticator.java
@@ -21,14 +21,17 @@ package org.apache.tinkerpop.gremlin.server.auth;
import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.CredentialTraversal;
import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.CredentialTraversalDsl;
import org.apache.tinkerpop.gremlin.groovy.jsr223.dsl.credential.CredentialTraversalSource;
+import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.mindrot.jbcrypt.BCrypt;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.net.InetAddress;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
@@ -80,12 +83,56 @@ public class SimpleAuthenticator implements Authenticator {
// have to create the indices because they are not stored in gryo
final TinkerGraph tinkerGraph = (TinkerGraph) graph;
tinkerGraph.createIndex(PROPERTY_USERNAME, Vertex.class);
+
+ // TinkerGraph no longer auto-loads from graphLocation on open, so read the credential store here from
+ // the configured location/format. TinkerStorageGraph (which persists via its own storage engine) manages
+ // its own data and is left untouched.
+ loadCredentialStore(tinkerGraph);
}
credentialStore = graph.traversal(CredentialTraversalSource.class);
logger.info("CredentialGraph initialized at {}", credentialStore);
}
+ /**
+ * Reads the credential store into the supplied in-memory {@link TinkerGraph} from the {@code graphLocation}
+ * declared in its configuration, if any. TinkerGraph no longer loads from disk automatically on open, so the
+ * credential file must be read explicitly here. A {@code TinkerStorageGraph} configured with a durable storage
+ * engine manages its own data and is skipped (it has no {@code graphFormat}).
+ */
+ private static void loadCredentialStore(final TinkerGraph graph) {
+ final Configuration conf = graph.configuration();
+ final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
+ // a storage engine manages its own persistence and is not an interchange-format load
+ final String storage = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE, null);
+ if (null == location || storage != null)
+ return;
+
+ final File f = new File(location);
+ if (!f.exists() || !f.isFile())
+ return;
+
+ final String format = conf.getString("gremlin.tinkergraph.graphFormat", "gryo");
+ try {
+ switch (format) {
+ case "graphml":
+ graph.io(IoCore.graphml()).readGraph(location);
+ break;
+ case "graphson":
+ graph.io(IoCore.graphson()).readGraph(location);
+ break;
+ case "gryo":
+ graph.io(IoCore.gryo()).readGraph(location);
+ break;
+ default:
+ graph.io(IoCore.createIoBuilder(format)).readGraph(location);
+ break;
+ }
+ } catch (Exception ex) {
+ throw new IllegalStateException(String.format("Could not load credential store at %s with format %s", location, format), ex);
+ }
+ }
+
@Override
public SaslNegotiator newSaslNegotiator(final InetAddress remoteAddress) {
return new PlainTextSaslAuthenticator();
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
index fac8d87b2d..ac1a3e6474 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
@@ -28,7 +28,6 @@ import org.apache.tinkerpop.gremlin.structure.Transaction;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.io.Io;
-import org.apache.tinkerpop.gremlin.structure.io.IoCore;
import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONVersion;
import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoVersion;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
@@ -36,8 +35,9 @@ import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComp
import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComputerView;
import org.apache.tinkerpop.gremlin.gql.GqlDeclarativeMatchStrategy;
import org.apache.tinkerpop.gremlin.tinkergraph.services.TinkerServiceRegistry;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.DefaultStorage;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorage;
-import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.util.Collections;
import java.util.Iterator;
@@ -77,7 +77,18 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
protected Configuration configuration;
protected String graphLocation;
- protected String graphFormat;
+
+ /**
+ * The pluggable durable storage engine, or {@code null} when the graph holds data only in memory. Only set by
+ * transactional implementations that support persistence.
+ */
+ protected TinkerStorage storage;
+
+ /**
+ * Guard set while a graph is replaying its storage log on open. While {@code true}, mutations must not be
+ * re-persisted, otherwise replay would append the loaded data back to the log.
+ */
+ protected volatile boolean loading = false;
/**
* {@inheritDoc}
@@ -243,54 +254,6 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
return vertexProperties.containsKey(id);
}
- protected void loadGraph() {
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) {
- try {
- if (graphFormat.equals("graphml")) {
- io(IoCore.graphml()).readGraph(graphLocation);
- } else if (graphFormat.equals("graphson")) {
- io(IoCore.graphson()).readGraph(graphLocation);
- } else if (graphFormat.equals("gryo")) {
- io(IoCore.gryo()).readGraph(graphLocation);
- } else {
- io(IoCore.createIoBuilder(graphFormat)).readGraph(graphLocation);
- }
- } catch (Exception ex) {
- throw new RuntimeException(String.format("Could not load graph at %s with %s", graphLocation, graphFormat), ex);
- }
- }
- }
-
- protected void saveGraph() {
- final File f = new File(graphLocation);
- if (f.exists()) {
- f.delete();
- } else {
- final File parent = f.getParentFile();
-
- // the parent would be null in the case of an relative path if the graphLocation was simply: "f.gryo"
- if (parent != null && !parent.exists()) {
- parent.mkdirs();
- }
- }
-
- try {
- if (graphFormat.equals("graphml")) {
- io(IoCore.graphml()).writeGraph(graphLocation);
- } else if (graphFormat.equals("graphson")) {
- io(IoCore.graphson()).writeGraph(graphLocation);
- } else if (graphFormat.equals("gryo")) {
- io(IoCore.gryo()).writeGraph(graphLocation);
- } else {
- io(IoCore.createIoBuilder(graphFormat)).writeGraph(graphLocation);
- }
- } catch (Exception ex) {
- throw new RuntimeException(String.format("Could not save graph at %s with %s", graphLocation, graphFormat), ex);
- }
- }
-
-
@Override
public <I extends Io> I io(final Io.Builder<I> builder) {
if (builder.requiresVersion(GryoVersion.V1_0) || builder.requiresVersion(GraphSONVersion.V1_0))
@@ -337,13 +300,18 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
}
/**
- * This method only has an effect if the {@link TinkerGraph#GREMLIN_TINKERGRAPH_GRAPH_LOCATION} is set, in which case the
- * data in the graph is persisted to that location. This method may be called multiple times and does not release
- * resources.
+ * Closes the graph, releasing any resources held by its {@link TinkerServiceRegistry}. This method may be called
+ * multiple times and is a no-op with respect to graph data for the in-memory implementation. Transactional
+ * implementations that are backed by a {@link org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorage}
+ * engine flush and close that engine here.
*/
@Override
public void close() {
- if (graphLocation != null) saveGraph();
+ if (storage != null) {
+ storage.flush();
+ storage.compact(this);
+ storage.close();
+ }
serviceRegistry.close();
GqlDeclarativeMatchStrategy.evict(this);
}
@@ -555,6 +523,28 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
}
}
+ ///////////// Storage engine ///////////////
+ /**
+ * Construct a {@link TinkerStorage} engine from the TinkerGraph {@code Configuration}, or return {@code null} when
+ * no storage engine is configured. The configuration value is either a {@link DefaultStorage} enum name (matched
+ * case-insensitively, e.g. {@code graphbinary}) or the fully-qualified class name of a {@link TinkerStorage}
+ * implementation with a public no-argument constructor. Mirrors {@link #selectIdManager}.
+ */
+ protected static TinkerStorage selectStorage(final Configuration config, final String configKey) {
+ final String storageConfigValue = config.getString(configKey, null);
+ if (null == storageConfigValue)
+ return null;
+ try {
+ return DefaultStorage.valueOf(storageConfigValue.toUpperCase()).get();
+ } catch (IllegalArgumentException iae) {
+ try {
+ return (TinkerStorage) Class.forName(storageConfigValue).newInstance();
+ } catch (Exception ex) {
+ throw new IllegalStateException(String.format("Could not configure TinkerGraph storage engine with %s", storageConfigValue), ex);
+ }
+ }
+ }
+
protected TinkerServiceRegistry.TinkerServiceFactory instantiate(final String className) {
try {
return (TinkerServiceRegistry.TinkerServiceFactory) Class.forName(className).getConstructor(AbstractTinkerGraph.class).newInstance(this);
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index dc931b9061..0935c38fdb 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -45,8 +45,20 @@ public interface TinkerGraph extends Graph {
String GREMLIN_TINKERGRAPH_EDGE_ID_MANAGER = "gremlin.tinkergraph.edgeIdManager";
String GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER = "gremlin.tinkergraph.vertexPropertyIdManager";
String GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY = "gremlin.tinkergraph.defaultVertexPropertyCardinality";
+ /**
+ * The filesystem directory that a {@link TinkerStorageGraph} uses for its durable storage engine. Ignored by
+ * {@link TinkerMemoryGraph}, which is purely in-memory. Only meaningful when {@link #GREMLIN_TINKERGRAPH_STORAGE}
+ * is also set.
+ */
String GREMLIN_TINKERGRAPH_GRAPH_LOCATION = "gremlin.tinkergraph.graphLocation";
- String GREMLIN_TINKERGRAPH_GRAPH_FORMAT = "gremlin.tinkergraph.graphFormat";
+ /**
+ * Selects the pluggable storage engine used by {@link TinkerStorageGraph} to durably persist transactions to the
+ * {@link #GREMLIN_TINKERGRAPH_GRAPH_LOCATION} directory. The value is either a
+ * {@code TinkerStorageGraph.DefaultStorage} enum name (e.g. {@code graphbinary}) or the fully-qualified class name
+ * of a {@code org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorage} implementation. When unset,
+ * the graph holds data only in memory. Not valid on {@link TinkerMemoryGraph}.
+ */
+ String GREMLIN_TINKERGRAPH_STORAGE = "gremlin.tinkergraph.storage";
String GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES = "gremlin.tinkergraph.allowNullPropertyValues";
String GREMLIN_TINKERGRAPH_SERVICE = "gremlin.tinkergraph.service";
String GREMLIN_TINKERGRAPH_VERTEX_LABEL_CARDINALITY = "gremlin.tinkergraph.vertexLabelCardinality";
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraph.java
index fc1afcad33..8dd7b1671f 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraph.java
@@ -50,8 +50,10 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
/**
- * The in-memory (with optional persistence on calls to {@link #close()}) implementation of the {@link TinkerGraph}
- * interface and the reference implementation of the property graph interfaces provided by TinkerPop.
+ * The purely in-memory implementation of the {@link TinkerGraph} interface and the reference implementation of the
+ * property graph interfaces provided by TinkerPop. This graph holds no data across JVM restarts; use
+ * {@code g.io(...).write()} / {@code g.io(...).read()} for interchange, or {@link TinkerStorageGraph} for durable
+ * persistence.
*
* @author Marko A. Rodriguez (http://markorodriguez.com)
* @author Stephen Mallette (http://stephen.genoprime.com)
@@ -99,15 +101,6 @@ public class TinkerMemoryGraph extends AbstractTinkerGraph {
defaultVertexLabel = Vertex.DEFAULT_LABEL;
defaultEdgeLabel = Edge.DEFAULT_LABEL;
- graphLocation = configuration.getString(GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
- graphFormat = configuration.getString(GREMLIN_TINKERGRAPH_GRAPH_FORMAT, null);
-
- if ((graphLocation != null && null == graphFormat) || (null == graphLocation && graphFormat != null))
- throw new IllegalStateException(String.format("The %s and %s must both be specified if either is present",
- GREMLIN_TINKERGRAPH_GRAPH_LOCATION, GREMLIN_TINKERGRAPH_GRAPH_FORMAT));
-
- if (graphLocation != null) loadGraph();
-
serviceRegistry = new TinkerServiceRegistry(this);
configuration.getList(String.class, GREMLIN_TINKERGRAPH_SERVICE, Collections.emptyList()).forEach(serviceClass ->
serviceRegistry.registerService(instantiate(serviceClass)));
@@ -447,6 +440,11 @@ public class TinkerMemoryGraph extends AbstractTinkerGraph {
return false;
}
+ @Override
+ public boolean supportsPersistence() {
+ return false;
+ }
+
@Override
public boolean supportsServiceCall() {
return true;
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
index c74c863e34..7dd54500a7 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
@@ -96,17 +96,25 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
defaultEdgeLabel = Edge.DEFAULT_LABEL;
graphLocation = configuration.getString(GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
- graphFormat = configuration.getString(GREMLIN_TINKERGRAPH_GRAPH_FORMAT, null);
+ storage = selectStorage(configuration, GREMLIN_TINKERGRAPH_STORAGE);
- if ((graphLocation != null && null == graphFormat) || (null == graphLocation && graphFormat != null))
- throw new IllegalStateException(String.format("The %s and %s must both be specified if either is present",
- GREMLIN_TINKERGRAPH_GRAPH_LOCATION, GREMLIN_TINKERGRAPH_GRAPH_FORMAT));
-
- if (graphLocation != null) loadGraph();
+ if (storage != null && null == graphLocation)
+ throw new IllegalStateException(String.format("The %s must be specified when %s is set",
+ GREMLIN_TINKERGRAPH_GRAPH_LOCATION, GREMLIN_TINKERGRAPH_STORAGE));
serviceRegistry = new TinkerServiceRegistry(this);
configuration.getList(String.class, GREMLIN_TINKERGRAPH_SERVICE, Collections.emptyList()).forEach(serviceClass ->
serviceRegistry.registerService(instantiate(serviceClass)));
+
+ if (storage != null) {
+ storage.open(this, configuration);
+ loading = true;
+ try {
+ storage.replay(this);
+ } finally {
+ loading = false;
+ }
+ }
}
/**
@@ -289,6 +297,17 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
this.edges.clear();
}
+ /**
+ * Fold the durable storage log into a compact snapshot of the current committed state, reclaiming space. Has no
+ * effect when no storage engine is configured.
+ */
+ public void compact() {
+ if (storage != null) {
+ storage.flush();
+ storage.compact(this);
+ }
+ }
+
@Override
public Transaction tx() {
return transaction;
@@ -519,6 +538,11 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
return true;
}
+ @Override
+ public boolean supportsPersistence() {
+ return storage != null;
+ }
+
@Override
public boolean supportsServiceCall() {
return true;
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
index 0bfcf09e46..7bef8a9264 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
@@ -21,9 +21,12 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure;
import org.apache.tinkerpop.gremlin.structure.Transaction;
import org.apache.tinkerpop.gremlin.structure.util.AbstractThreadLocalTransaction;
import org.apache.tinkerpop.gremlin.structure.util.TransactionException;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorageMutation;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
@@ -174,6 +177,14 @@ final class TinkerTransaction extends AbstractThreadLocalTransaction {
final TinkerTransactionalIndex edgeIndex = (TinkerTransactionalIndex) graph.edgeIndex;
if (edgeIndex != null) edgeIndex.commit(changedEdges);
+ // write-ahead: durably persist the changeset before applying the in-memory commit, so a failure here
+ // aborts the commit (via the catch below) and leaves memory and disk consistent. Skipped while the graph
+ // is replaying its storage log on open.
+ if (graph.storage != null && !graph.loading) {
+ graph.storage.persist(txVersion, toVertexMutations(changedVertices), toEdgeMutations(changedEdges));
+ graph.storage.flush();
+ }
+
// commit all changes
changedVertices.forEach(v -> v.commit(txVersion));
changedEdges.forEach(e -> e.commit(txVersion));
@@ -209,6 +220,34 @@ final class TinkerTransaction extends AbstractThreadLocalTransaction {
}
}
+ /**
+ * Convert the changed vertex containers into the storage-facing {@link TinkerStorageMutation} view. Called during
+ * commit, before {@code commit()} is applied to the containers, so the modified value is still available via
+ * {@link TinkerElementContainer#getModified()}.
+ */
+ private static List<TinkerStorageMutation<TinkerVertex>> toVertexMutations(final Set<TinkerElementContainer<TinkerVertex>> changed) {
+ final List<TinkerStorageMutation<TinkerVertex>> mutations = new ArrayList<>(changed.size());
+ for (final TinkerElementContainer<TinkerVertex> c : changed) {
+ mutations.add(c.isDeleted()
+ ? new TinkerStorageMutation<>(c.getElementId(), null)
+ : new TinkerStorageMutation<>(c.getElementId(), c.getModified()));
+ }
+ return mutations;
+ }
+
+ /**
+ * Convert the changed edge containers into the storage-facing {@link TinkerStorageMutation} view.
+ */
+ private static List<TinkerStorageMutation<TinkerEdge>> toEdgeMutations(final Set<TinkerElementContainer<TinkerEdge>> changed) {
+ final List<TinkerStorageMutation<TinkerEdge>> mutations = new ArrayList<>(changed.size());
+ for (final TinkerElementContainer<TinkerEdge> c : changed) {
+ mutations.add(c.isDeleted()
+ ? new TinkerStorageMutation<>(c.getElementId(), null)
+ : new TinkerStorageMutation<>(c.getElementId(), c.getModified()));
+ }
+ return mutations;
+ }
+
/**
* Rollback all changes made in current transaction.
* Workflow:
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ByteBufferBuffer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ByteBufferBuffer.java
new file mode 100644
index 0000000000..8872afd404
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ByteBufferBuffer.java
@@ -0,0 +1,336 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.tinkerpop.gremlin.structure.io.Buffer;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+/**
+ * A self-contained, heap {@code byte[]}-backed implementation of the gremlin-core {@link Buffer} abstraction, used to
+ * drive {@code GraphBinaryWriter}/{@code GraphBinaryReader} without depending on gremlin-util's Netty-backed buffer.
+ * All multi-byte values are big-endian, matching the wire order the Netty implementation uses. The backing array grows
+ * as needed on write. This class is not thread-safe; a buffer is used by a single thread for a single
+ * serialize/deserialize.
+ */
+public final class ByteBufferBuffer implements Buffer {
+
+ private static final int DEFAULT_CAPACITY = 256;
+
+ private byte[] array;
+ private int readerIndex = 0;
+ private int writerIndex = 0;
+ private int markedWriterIndex = 0;
+ private int referenceCount = 1;
+
+ public ByteBufferBuffer() {
+ this(DEFAULT_CAPACITY);
+ }
+
+ public ByteBufferBuffer(final int initialCapacity) {
+ this.array = new byte[Math.max(initialCapacity, 1)];
+ }
+
+ /**
+ * Wraps an existing array for reading. The writer index is positioned at the end of the supplied data.
+ */
+ public ByteBufferBuffer(final byte[] data) {
+ this.array = data;
+ this.writerIndex = data.length;
+ }
+
+ /**
+ * Returns a copy of the readable-region bytes (from reader index to writer index). Does not change indexes.
+ */
+ public byte[] toReadableArray() {
+ return Arrays.copyOfRange(array, readerIndex, writerIndex);
+ }
+
+ /**
+ * Returns a copy of all written bytes (index 0 to writer index). Does not change indexes.
+ */
+ public byte[] toWrittenArray() {
+ return Arrays.copyOfRange(array, 0, writerIndex);
+ }
+
+ private void ensureWritable(final int additional) {
+ final int required = writerIndex + additional;
+ if (required <= array.length)
+ return;
+ int newCapacity = array.length;
+ while (newCapacity < required)
+ newCapacity <<= 1;
+ array = Arrays.copyOf(array, newCapacity);
+ }
+
+ private void checkReadable(final int length) {
+ if (readerIndex + length > writerIndex)
+ throw new IndexOutOfBoundsException(String.format(
+ "Not enough readable bytes: need %d at index %d but writer index is %d", length, readerIndex, writerIndex));
+ }
+
+ @Override
+ public int readableBytes() {
+ return writerIndex - readerIndex;
+ }
+
+ @Override
+ public int readerIndex() {
+ return readerIndex;
+ }
+
+ @Override
+ public Buffer readerIndex(final int readerIndex) {
+ if (readerIndex < 0 || readerIndex > writerIndex)
+ throw new IndexOutOfBoundsException("readerIndex: " + readerIndex);
+ this.readerIndex = readerIndex;
+ return this;
+ }
+
+ @Override
+ public int writerIndex() {
+ return writerIndex;
+ }
+
+ @Override
+ public Buffer writerIndex(final int writerIndex) {
+ if (writerIndex < readerIndex)
+ throw new IndexOutOfBoundsException("writerIndex: " + writerIndex);
+ ensureWritable(writerIndex - this.writerIndex);
+ this.writerIndex = writerIndex;
+ return this;
+ }
+
+ @Override
+ public Buffer markWriterIndex() {
+ this.markedWriterIndex = writerIndex;
+ return this;
+ }
+
+ @Override
+ public Buffer resetWriterIndex() {
+ this.writerIndex = markedWriterIndex;
+ return this;
+ }
+
+ @Override
+ public int capacity() {
+ return array.length;
+ }
+
+ @Override
+ public boolean isDirect() {
+ return false;
+ }
+
+ @Override
+ public boolean readBoolean() {
+ return readByte() != 0;
+ }
+
+ @Override
+ public byte readByte() {
+ checkReadable(1);
+ return array[readerIndex++];
+ }
+
+ @Override
+ public short readShort() {
+ checkReadable(2);
+ return (short) (((array[readerIndex++] & 0xFF) << 8) | (array[readerIndex++] & 0xFF));
+ }
+
+ @Override
+ public int readInt() {
+ checkReadable(4);
+ return ((array[readerIndex++] & 0xFF) << 24) |
+ ((array[readerIndex++] & 0xFF) << 16) |
+ ((array[readerIndex++] & 0xFF) << 8) |
+ (array[readerIndex++] & 0xFF);
+ }
+
+ @Override
+ public long readLong() {
+ checkReadable(8);
+ long value = 0;
+ for (int i = 0; i < 8; i++)
+ value = (value << 8) | (array[readerIndex++] & 0xFF);
+ return value;
+ }
+
+ @Override
+ public float readFloat() {
+ return Float.intBitsToFloat(readInt());
+ }
+
+ @Override
+ public double readDouble() {
+ return Double.longBitsToDouble(readLong());
+ }
+
+ @Override
+ public Buffer readBytes(final byte[] destination) {
+ return readBytes(destination, 0, destination.length);
+ }
+
+ @Override
+ public Buffer readBytes(final byte[] destination, final int dstIndex, final int length) {
+ checkReadable(length);
+ System.arraycopy(array, readerIndex, destination, dstIndex, length);
+ readerIndex += length;
+ return this;
+ }
+
+ @Override
+ public Buffer readBytes(final ByteBuffer dst) {
+ final int length = dst.remaining();
+ checkReadable(length);
+ dst.put(array, readerIndex, length);
+ readerIndex += length;
+ return this;
+ }
+
+ @Override
+ public Buffer readBytes(final OutputStream out, final int length) throws IOException {
+ checkReadable(length);
+ out.write(array, readerIndex, length);
+ readerIndex += length;
+ return this;
+ }
+
+ @Override
+ public Buffer writeBoolean(final boolean value) {
+ return writeByte(value ? 1 : 0);
+ }
+
+ @Override
+ public Buffer writeByte(final int value) {
+ ensureWritable(1);
+ array[writerIndex++] = (byte) value;
+ return this;
+ }
+
+ @Override
+ public Buffer writeShort(final int value) {
+ ensureWritable(2);
+ array[writerIndex++] = (byte) (value >>> 8);
+ array[writerIndex++] = (byte) value;
+ return this;
+ }
+
+ @Override
+ public Buffer writeInt(final int value) {
+ ensureWritable(4);
+ array[writerIndex++] = (byte) (value >>> 24);
+ array[writerIndex++] = (byte) (value >>> 16);
+ array[writerIndex++] = (byte) (value >>> 8);
+ array[writerIndex++] = (byte) value;
+ return this;
+ }
+
+ @Override
+ public Buffer writeLong(final long value) {
+ ensureWritable(8);
+ for (int i = 56; i >= 0; i -= 8)
+ array[writerIndex++] = (byte) (value >>> i);
+ return this;
+ }
+
+ @Override
+ public Buffer writeFloat(final float value) {
+ return writeInt(Float.floatToIntBits(value));
+ }
+
+ @Override
+ public Buffer writeDouble(final double value) {
+ return writeLong(Double.doubleToLongBits(value));
+ }
+
+ @Override
+ public Buffer writeBytes(final byte[] src) {
+ return writeBytes(src, 0, src.length);
+ }
+
+ @Override
+ public Buffer writeBytes(final ByteBuffer src) {
+ final int length = src.remaining();
+ ensureWritable(length);
+ src.get(array, writerIndex, length);
+ writerIndex += length;
+ return this;
+ }
+
+ @Override
+ public Buffer writeBytes(final byte[] src, final int srcIndex, final int length) {
+ ensureWritable(length);
+ System.arraycopy(src, srcIndex, array, writerIndex, length);
+ writerIndex += length;
+ return this;
+ }
+
+ @Override
+ public boolean release() {
+ return --referenceCount <= 0;
+ }
+
+ @Override
+ public Buffer retain() {
+ referenceCount++;
+ return this;
+ }
+
+ @Override
+ public int referenceCount() {
+ return referenceCount;
+ }
+
+ @Override
+ public int nioBufferCount() {
+ return 1;
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers() {
+ return new ByteBuffer[] { nioBuffer() };
+ }
+
+ @Override
+ public ByteBuffer[] nioBuffers(final int index, final int length) {
+ return new ByteBuffer[] { nioBuffer(index, length) };
+ }
+
+ @Override
+ public ByteBuffer nioBuffer() {
+ return nioBuffer(readerIndex, readableBytes());
+ }
+
+ @Override
+ public ByteBuffer nioBuffer(final int index, final int length) {
+ return ByteBuffer.wrap(Arrays.copyOfRange(array, index, index + length));
+ }
+
+ @Override
+ public Buffer getBytes(final int index, final byte[] dst) {
+ System.arraycopy(array, index, dst, 0, dst.length);
+ return this;
+ }
+}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DefaultStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DefaultStorage.java
new file mode 100644
index 0000000000..49c4a2becf
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/DefaultStorage.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import java.util.function.Supplier;
+
+/**
+ * The built-in {@link TinkerStorage} engines that can be selected by name via the
+ * {@code gremlin.tinkergraph.storage} configuration key. A fully-qualified class name may be used instead of one of
+ * these names to plug in a custom engine.
+ */
+public enum DefaultStorage implements Supplier<TinkerStorage> {
+
+ /**
+ * A durable, append-only commit log ("write-ahead log") that serializes each committed transaction with
+ * GraphBinary. See {@link GraphBinaryStorage}.
+ */
+ GRAPHBINARY {
+ @Override
+ public TinkerStorage get() {
+ return new GraphBinaryStorage();
+ }
+ }
+}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
new file mode 100644
index 0000000000..b07891a93c
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -0,0 +1,379 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader;
+import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter;
+import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry;
+import org.apache.tinkerpop.gremlin.structure.util.Attachable;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedFactory;
+import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
+
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A durable {@link TinkerStorage} engine that persists a {@code TinkerStorageGraph} as an append-only commit log
+ * ("write-ahead log") serialized with GraphBinary. On each committed transaction the changeset is appended to
+ * {@code log.gbin} as a single record; on open the optional {@code snapshot.gbin} is read followed by the log, with the
+ * folded result re-applied to the in-memory graph. {@link #compact(AbstractTinkerGraph)} rewrites the snapshot from the
+ * current committed state and truncates the log.
+ * <p/>
+ * The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
+ */
+public final class GraphBinaryStorage implements TinkerStorage {
+
+ /**
+ * Version byte prefixing every record, allowing the on-disk format to evolve.
+ */
+ static final byte FORMAT_VERSION = 1;
+
+ private static final byte OP_PUT_VERTEX = 1;
+ private static final byte OP_DEL_VERTEX = 2;
+ private static final byte OP_PUT_EDGE = 3;
+ private static final byte OP_DEL_EDGE = 4;
+
+ static final String SNAPSHOT_FILE = "snapshot.gbin";
+ static final String LOG_FILE = "log.gbin";
+
+ private final GraphBinaryWriter writer = new GraphBinaryWriter(TypeSerializerRegistry.INSTANCE);
+ private final GraphBinaryReader reader = new GraphBinaryReader(TypeSerializerRegistry.INSTANCE);
+
+ private File directory;
+ private File snapshotFile;
+ private File logFile;
+
+ private DataOutputStream logOut;
+ private boolean closed = false;
+
+ @Override
+ public void open(final AbstractTinkerGraph graph, final Configuration config) {
+ final String location = config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null);
+ if (null == location)
+ throw new IllegalStateException(String.format("%s must be set to use the GraphBinary storage engine",
+ TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION));
+ this.directory = new File(location);
+ this.snapshotFile = new File(directory, SNAPSHOT_FILE);
+ this.logFile = new File(directory, LOG_FILE);
+ ensureDirectory();
+ }
+
+ /**
+ * Ensure the backing directory exists, creating it if necessary. Called on open and again before writing a
+ * snapshot, since {@code close()} may be invoked more than once and the directory may have been removed in between.
+ */
+ private void ensureDirectory() {
+ if (directory.exists()) {
+ if (!directory.isDirectory())
+ throw new IllegalStateException(String.format("Storage location %s exists but is not a directory", directory));
+ } else if (!directory.mkdirs()) {
+ throw new IllegalStateException(String.format("Could not create storage directory %s", directory));
+ }
+ }
+
+ @Override
+ public void replay(final AbstractTinkerGraph graph) {
+ // Fold snapshot then log into final state: last write per id wins, deletes remove.
+ final Map<Object, DetachedVertex> vertices = new LinkedHashMap<>();
+ final Map<Object, DetachedEdge> edges = new LinkedHashMap<>();
+
+ if (snapshotFile.exists())
+ foldRecords(snapshotFile, vertices, edges);
+ if (logFile.exists())
+ foldRecords(logFile, vertices, edges);
+
+ if (vertices.isEmpty() && edges.isEmpty())
+ return;
+
+ // Attach vertices first so edges can find their endpoints, then commit once.
+ for (final DetachedVertex v : vertices.values())
+ v.attach(Attachable.Method.getOrCreate(graph));
+ for (final DetachedEdge e : edges.values())
+ e.attach(Attachable.Method.getOrCreate(graph));
+
+ graph.tx().commit();
+ }
+
+ /**
+ * Read every record in a file, folding puts and deletes into the supplied maps.
+ */
+ private void foldRecords(final File file, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) {
+ try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(file)))) {
+ while (true) {
+ final byte[] record;
+ try {
+ record = readFrame(in);
+ } catch (EOFException eof) {
+ break;
+ }
+ if (record == null)
+ break;
+ applyRecord(record, vertices, edges);
+ }
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not read storage file %s", file), ex);
+ }
+ }
+
+ private void applyRecord(final byte[] record, final Map<Object, DetachedVertex> vertices, final Map<Object, DetachedEdge> edges) throws IOException {
+ final ByteBufferBuffer buffer = new ByteBufferBuffer(record);
+ final byte version = buffer.readByte();
+ if (version != FORMAT_VERSION)
+ throw new IOException(String.format("Unsupported storage record version %d (expected %d)", version, FORMAT_VERSION));
+ buffer.readLong(); // txVersion, retained for diagnostics/future use
+ final int entryCount = buffer.readInt();
+ for (int i = 0; i < entryCount; i++) {
+ final byte op = buffer.readByte();
+ switch (op) {
+ case OP_PUT_VERTEX: {
+ final Vertex v = reader.read(buffer);
+ vertices.put(v.id(), (DetachedVertex) v);
+ break;
+ }
+ case OP_DEL_VERTEX: {
+ final Object id = reader.read(buffer);
+ vertices.remove(id);
+ break;
+ }
+ case OP_PUT_EDGE: {
+ final Edge e = reader.read(buffer);
+ edges.put(e.id(), (DetachedEdge) e);
+ break;
+ }
+ case OP_DEL_EDGE: {
+ final Object id = reader.read(buffer);
+ edges.remove(id);
+ break;
+ }
+ default:
+ throw new IOException("Unknown storage op code: " + op);
+ }
+ }
+ }
+
+ @Override
+ public void persist(final long txVersion,
+ final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) {
+ ensureLogOpen();
+ try {
+ final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges);
+ writeFrame(logOut, frame);
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not append transaction to storage log", ex);
+ }
+ }
+
+ /**
+ * Serialize a commit record: version byte, txVersion, entry count, then each entry as an op byte followed by
+ * either the serialized element (put) or the serialized id (delete).
+ */
+ private byte[] encodeRecord(final long txVersion,
+ final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) throws IOException {
+ final ByteBufferBuffer buffer = new ByteBufferBuffer();
+ buffer.writeByte(FORMAT_VERSION);
+ buffer.writeLong(txVersion);
+ buffer.writeInt(changedVertices.size() + changedEdges.size());
+ for (final TinkerStorageMutation<TinkerVertex> m : changedVertices) {
+ if (m.isDeleted()) {
+ buffer.writeByte(OP_DEL_VERTEX);
+ writer.write(m.id(), buffer);
+ } else {
+ buffer.writeByte(OP_PUT_VERTEX);
+ // detach to a stable form independent of the transactional element
+ writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ }
+ }
+ for (final TinkerStorageMutation<TinkerEdge> m : changedEdges) {
+ if (m.isDeleted()) {
+ buffer.writeByte(OP_DEL_EDGE);
+ writer.write(m.id(), buffer);
+ } else {
+ buffer.writeByte(OP_PUT_EDGE);
+ writer.write(DetachedFactory.detach(m.element(), true), buffer);
+ }
+ }
+ return buffer.toWrittenArray();
+ }
+
+ @Override
+ public void flush() {
+ if (closed)
+ return;
+ if (logOut != null) {
+ try {
+ logOut.flush();
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not flush storage log", ex);
+ }
+ }
+ }
+
+ @Override
+ public void compact(final AbstractTinkerGraph graph) {
+ if (closed)
+ return;
+ // Write a fresh snapshot of the current committed state, then truncate the log.
+ closeLog();
+ ensureDirectory();
+ final File tmp = new File(directory, SNAPSHOT_FILE + ".tmp");
+ try (final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) {
+ final byte[] frame = encodeSnapshot(graph);
+ if (frame.length > 0)
+ writeFrame(out, frame);
+ out.flush();
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not write storage snapshot", ex);
+ }
+
+ if (snapshotFile.exists() && !snapshotFile.delete())
+ throw new UncheckedIOException(new IOException("Could not replace snapshot " + snapshotFile));
+ if (!tmp.renameTo(snapshotFile))
+ throw new UncheckedIOException(new IOException("Could not rename snapshot into place " + snapshotFile));
+
+ // truncate the log
+ if (logFile.exists() && !logFile.delete())
+ throw new UncheckedIOException(new IOException("Could not truncate storage log " + logFile));
+ }
+
+ /**
+ * Serialize the entire current committed state of the graph as a single put-only record.
+ */
+ private byte[] encodeSnapshot(final AbstractTinkerGraph graph) throws IOException {
+ final List<Vertex> vertexList = new ArrayList<>();
+ final Iterator<Vertex> vertexIterator = graph.vertices();
+ while (vertexIterator.hasNext())
+ vertexList.add(vertexIterator.next());
+ final List<Edge> edgeList = new ArrayList<>();
+ final Iterator<Edge> edgeIterator = graph.edges();
+ while (edgeIterator.hasNext())
+ edgeList.add(edgeIterator.next());
+
+ if (vertexList.isEmpty() && edgeList.isEmpty())
+ return new byte[0];
+
+ final ByteBufferBuffer buffer = new ByteBufferBuffer();
+ buffer.writeByte(FORMAT_VERSION);
+ buffer.writeLong(0L); // snapshot has no single tx version
+ buffer.writeInt(vertexList.size() + edgeList.size());
+ for (final Vertex v : vertexList) {
+ buffer.writeByte(OP_PUT_VERTEX);
+ writer.write(DetachedFactory.detach(v, true), buffer);
+ }
+ for (final Edge e : edgeList) {
+ buffer.writeByte(OP_PUT_EDGE);
+ writer.write(DetachedFactory.detach(e, true), buffer);
+ }
+ return buffer.toWrittenArray();
+ }
+
+ @Override
+ public void close() {
+ closeLog();
+ closed = true;
+ }
+
+ private void ensureLogOpen() {
+ if (logOut == null) {
+ try {
+ logOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(logFile, true)));
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not open storage log for append", ex);
+ }
+ }
+ }
+
+ private void closeLog() {
+ if (logOut != null) {
+ try {
+ logOut.flush();
+ logOut.close();
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not close storage log", ex);
+ } finally {
+ logOut = null;
+ }
+ }
+ }
+
+ /**
+ * Write a length-prefixed frame: a 4-byte big-endian length followed by the payload.
+ */
+ private static void writeFrame(final DataOutputStream out, final byte[] payload) throws IOException {
+ out.writeInt(payload.length);
+ out.write(payload);
+ }
+
+ /**
+ * Read a length-prefixed frame, or return {@code null} on a clean end of file. A truncated final frame (from a
+ * crash mid-append) is treated as end of file so earlier committed records still load.
+ */
+ private static byte[] readFrame(final DataInputStream in) throws IOException {
+ final int length;
+ try {
+ length = in.readInt();
+ } catch (EOFException eof) {
+ return null;
+ }
+ if (length < 0)
+ throw new IOException("Corrupt storage frame length: " + length);
+ final byte[] payload = new byte[length];
+ try {
+ readFully(in, payload);
+ } catch (EOFException eof) {
+ // partial trailing frame from an interrupted append — stop here
+ return null;
+ }
+ return payload;
+ }
+
+ private static void readFully(final InputStream in, final byte[] dst) throws IOException {
+ int off = 0;
+ while (off < dst.length) {
+ final int read = in.read(dst, off, dst.length - off);
+ if (read < 0)
+ throw new EOFException();
+ off += read;
+ }
+ }
+}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
new file mode 100644
index 0000000000..49dcedb1ba
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
+
+import java.util.Collection;
+
+/**
+ * A pluggable durable storage engine for a transactional {@code TinkerStorageGraph}. Implementations persist the
+ * changeset of each committed transaction to disk and rebuild the in-memory graph on open. The graph remains the
+ * authoritative in-memory copy (write-through); the engine is a durable mirror.
+ * <p/>
+ * The lifecycle is:
+ * <ol>
+ * <li>{@link #open(AbstractTinkerGraph, Configuration)} — resolve the backing location and ready the store.</li>
+ * <li>{@link #replay(AbstractTinkerGraph)} — rebuild in-memory state from what was previously persisted.</li>
+ * <li>{@link #persist(long, Collection, Collection)} then {@link #flush()} — called on each transaction commit,
+ * before the in-memory state is committed, so a failure aborts the commit and leaves memory and disk consistent.</li>
+ * <li>{@link #compact(AbstractTinkerGraph)} — optionally fold accumulated changes into a compact snapshot.</li>
+ * <li>{@link #close()} — release resources.</li>
+ * </ol>
+ * A new engine can be selected by name (see {@code TinkerStorageGraph.DefaultStorage}) or by fully-qualified class
+ * name via the {@code gremlin.tinkergraph.storage} configuration key. Implementations must provide a public no-argument
+ * constructor.
+ */
+public interface TinkerStorage extends AutoCloseable {
+
+ /**
+ * Prepare the storage engine for use, resolving its backing location from the supplied configuration. Called once
+ * during graph construction before {@link #replay(AbstractTinkerGraph)}.
+ *
+ * @param graph the graph that owns this engine
+ * @param config the graph configuration, including {@code gremlin.tinkergraph.graphLocation}
+ */
+ void open(AbstractTinkerGraph graph, Configuration config);
+
+ /**
+ * Rebuild the in-memory state of the graph from previously persisted data. Called once during graph construction.
+ * Implementations should re-apply persisted elements through the graph's own mutation API; the graph sets its
+ * {@code loading} guard for the duration so this does not re-persist.
+ *
+ * @param graph the graph to populate
+ */
+ void replay(AbstractTinkerGraph graph);
+
+ /**
+ * Durably record the changeset of a committing transaction. Called from within the transaction commit, while the
+ * changed elements are locked, before the in-memory commit is applied. Each mutation is either a put (added or
+ * modified element) or a delete (see {@link TinkerStorageMutation}).
+ *
+ * @param txVersion the version number of the committing transaction
+ * @param changedVertices the vertex mutations in this transaction
+ * @param changedEdges the edge mutations in this transaction
+ */
+ void persist(long txVersion,
+ Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ Collection<TinkerStorageMutation<TinkerEdge>> changedEdges);
+
+ /**
+ * Force any buffered writes to durable storage. Called after {@link #persist(long, Collection, Collection)} as the
+ * commit's durability point.
+ */
+ void flush();
+
+ /**
+ * Fold accumulated changes into a compact representation of the current committed state, reclaiming space. Safe to
+ * call at any time; typically invoked on {@link #close()} or on a size/commit-count threshold.
+ *
+ * @param graph the graph whose current committed state should be snapshotted
+ */
+ void compact(AbstractTinkerGraph graph);
+
+ /**
+ * {@inheritDoc}
+ * <p/>
+ * Flush and release resources. Does not delete persisted data.
+ */
+ @Override
+ void close();
+}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorageMutation.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorageMutation.java
new file mode 100644
index 0000000000..726c6862fd
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorageMutation.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.tinkerpop.gremlin.structure.Element;
+
+/**
+ * A single element change within a committing transaction, handed to a {@link TinkerStorage} engine to persist. This
+ * is the stable, public view of a change that decouples storage engines from TinkerGraph's internal transactional
+ * containers. A mutation is either a <em>put</em> (the element was added or modified, {@link #element()} is non-null)
+ * or a <em>delete</em> ({@link #element()} is {@code null} and {@link #isDeleted()} is {@code true}).
+ *
+ * @param <T> the element type ({@code TinkerVertex} or {@code TinkerEdge})
+ */
+public final class TinkerStorageMutation<T extends Element> {
+
+ private final Object id;
+ private final T element;
+
+ /**
+ * Create a mutation for the given element id.
+ *
+ * @param id the element identifier (never {@code null})
+ * @param element the committed element for a put, or {@code null} for a delete
+ */
+ public TinkerStorageMutation(final Object id, final T element) {
+ this.id = id;
+ this.element = element;
+ }
+
+ /**
+ * The identifier of the changed element.
+ */
+ public Object id() {
+ return id;
+ }
+
+ /**
+ * The committed element to persist, or {@code null} when this mutation is a deletion.
+ */
+ public T element() {
+ return element;
+ }
+
+ /**
+ * Returns {@code true} when this mutation deletes the element rather than adding or modifying it.
+ */
+ public boolean isDeleted() {
+ return element == null;
+ }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphProvider.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphProvider.java
index fe17fd7d71..170b43f8f5 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphProvider.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphProvider.java
@@ -21,7 +21,6 @@ package org.apache.tinkerpop.gremlin.tinkergraph;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.AbstractGraphProvider;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.GraphTest;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
@@ -39,7 +38,6 @@ import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertexProperty;
-import java.io.File;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -73,10 +71,6 @@ public class TinkerMemoryGraphProvider extends AbstractGraphProvider {
put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER, idMaker);
if (requiresListCardinalityAsDefault(loadGraphWith, test, testMethodName))
put(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name());
- if (requiresPersistence(test, testMethodName)) {
- put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,TestHelper.makeTestDataFile(test, "temp", testMethodName + ".kryo"));
- }
}};
}
@@ -84,13 +78,6 @@ public class TinkerMemoryGraphProvider extends AbstractGraphProvider {
public void clear(final Graph graph, final Configuration configuration) throws Exception {
if (graph != null)
graph.close();
-
- // in the even the graph is persisted we need to clean up
- final String graphLocation = null != configuration ? configuration.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null) : null;
- if (graphLocation != null) {
- final File f = new File(graphLocation);
- f.delete();
- }
}
@Override
@@ -98,13 +85,6 @@ public class TinkerMemoryGraphProvider extends AbstractGraphProvider {
return IMPLEMENTATION;
}
- /**
- * Determines if a test requires TinkerGraph persistence to be configured with graph location and format.
- */
- protected static boolean requiresPersistence(final Class<?> test, final String testMethodName) {
- return test == GraphTest.class && testMethodName.equals("shouldPersistDataOnClose");
- }
-
/**
* Determines if a test requires a different cardinality as the default or not.
*/
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphUUIDProvider.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphUUIDProvider.java
index 8a127e8330..ef37147838 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphUUIDProvider.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerMemoryGraphUUIDProvider.java
@@ -20,13 +20,11 @@
package org.apache.tinkerpop.gremlin.tinkergraph;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerMemoryGraph;
-import java.io.File;
import java.util.HashMap;
import java.util.Map;
@@ -47,10 +45,6 @@ public class TinkerMemoryGraphUUIDProvider extends TinkerMemoryGraphProvider {
put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER, idMaker);
if (requiresListCardinalityAsDefault(loadGraphWith, test, testMethodName))
put(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name());
- if (requiresPersistence(test, testMethodName)) {
- put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, TestHelper.makeTestDataFile(test, "temp", testMethodName + ".kryo"));
- }
}};
}
}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerStorageGraphProvider.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerStorageGraphProvider.java
index 6f777a630c..a0b686266f 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerStorageGraphProvider.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/TinkerStorageGraphProvider.java
@@ -70,8 +70,9 @@ public class TinkerStorageGraphProvider extends AbstractGraphProvider {
if (requiresListCardinalityAsDefault(loadGraphWith, test, testMethodName))
put(TinkerStorageGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name());
if (requiresPersistence(test, testMethodName)) {
- put(TinkerStorageGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- put(TinkerStorageGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,TestHelper.makeTestDataFile(test, "temp", testMethodName + ".kryo"));
+ put(TinkerStorageGraph.GREMLIN_TINKERGRAPH_STORAGE, "graphbinary");
+ put(TinkerStorageGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION,
+ TestHelper.makeTestDataDirectory(test, "temp", testMethodName));
}
}};
}
@@ -81,14 +82,26 @@ public class TinkerStorageGraphProvider extends AbstractGraphProvider {
if (graph != null)
graph.close();
- // in the even the graph is persisted we need to clean up
+ // in the event the graph is persisted we need to clean up the storage directory
final String graphLocation = null != configuration ? configuration.getString(TinkerStorageGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null) : null;
if (graphLocation != null) {
- final File f = new File(graphLocation);
- f.delete();
+ deleteRecursively(new File(graphLocation));
}
}
+ private static void deleteRecursively(final File file) {
+ if (!file.exists())
+ return;
+ if (file.isDirectory()) {
+ final File[] children = file.listFiles();
+ if (children != null) {
+ for (final File child : children)
+ deleteRecursively(child);
+ }
+ }
+ file.delete();
+ }
+
@Override
public Set<Class> getImplementations() {
return IMPLEMENTATION;
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraphTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraphTest.java
index 536f5b5ee8..95856f6020 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraphTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerMemoryGraphTest.java
@@ -21,7 +21,6 @@ package org.apache.tinkerpop.gremlin.tinkergraph.structure;
import org.apache.commons.configuration2.BaseConfiguration;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.GraphHelper;
-import org.apache.tinkerpop.gremlin.TestHelper;
import org.apache.tinkerpop.gremlin.process.computer.Computer;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
@@ -39,7 +38,6 @@ import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
-import org.apache.tinkerpop.gremlin.structure.io.Io;
import org.apache.tinkerpop.gremlin.structure.io.GraphReader;
import org.apache.tinkerpop.gremlin.structure.io.GraphWriter;
import org.apache.tinkerpop.gremlin.structure.io.IoCore;
@@ -63,11 +61,8 @@ import org.apache.tinkerpop.shaded.kryo.io.Output;
import org.junit.Test;
import java.awt.Color;
-import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
@@ -80,7 +75,6 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
import java.util.function.Supplier;
import static org.apache.tinkerpop.gremlin.process.traversal.AnonymousTraversalSource.traversal;
@@ -95,7 +89,6 @@ import static org.junit.Assert.assertThrows;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.junit.Assume.assumeThat;
-import static org.mockito.Mockito.mock;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
@@ -390,13 +383,6 @@ public class TinkerMemoryGraphTest {
}
}
- @Test(expected = IllegalStateException.class)
- public void shouldRequireGraphLocationIfFormatIsSet() {
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "graphml");
- TinkerGraph.open(conf);
- }
-
@Test(expected = IllegalStateException.class)
public void shouldNotModifyAVertexThatWasRemoved() {
final TinkerGraph graph = TinkerGraph.open();
@@ -431,137 +417,6 @@ public class TinkerMemoryGraphTest {
v.value("name");
}
- @Test(expected = IllegalStateException.class)
- public void shouldRequireGraphFormatIfLocationIsSet() {
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, TestHelper.makeTestDataDirectory(TinkerMemoryGraphTest.class));
- TinkerGraph.open(conf);
- }
-
- @Test
- public void shouldPersistToGraphML() {
- final String graphLocation = TestHelper.makeTestDataFile(TinkerMemoryGraphTest.class, "shouldPersistToGraphML.xml");
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "graphml");
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateModern(graph);
- graph.close();
-
- final TinkerGraph reloadedGraph = TinkerGraph.open(conf);
- IoTest.assertModernGraph(reloadedGraph, true, true);
- reloadedGraph.close();
- }
-
- @Test
- public void shouldPersistToGraphSON() {
- final String graphLocation = TestHelper.makeTestDataFile(TinkerMemoryGraphTest.class, "shouldPersistToGraphSON.json");
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "graphson");
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateModern(graph);
- graph.close();
-
- final TinkerGraph reloadedGraph = TinkerGraph.open(conf);
- IoTest.assertModernGraph(reloadedGraph, true, false);
- reloadedGraph.close();
- }
-
- @Test
- public void shouldPersistToGryo() {
- final String graphLocation = TestHelper.makeTestDataFile(TinkerMemoryGraphTest.class, "shouldPersistToGryo.kryo");
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateModern(graph);
- graph.close();
-
- final TinkerGraph reloadedGraph = TinkerGraph.open(conf);
- IoTest.assertModernGraph(reloadedGraph, true, false);
- reloadedGraph.close();
- }
-
- @Test
- public void shouldPersistToGryoAndHandleMultiProperties() {
- final String graphLocation = TestHelper.makeTestDataFile(TinkerMemoryGraphTest.class, "shouldPersistToGryoMulti.kryo");
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateTheCrew(graph);
- graph.close();
-
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.toString());
- final TinkerGraph reloadedGraph = TinkerGraph.open(conf);
- IoTest.assertCrewGraph(reloadedGraph, false);
- reloadedGraph.close();
- }
-
- @Test
- public void shouldPersistWithRelativePath() {
- final String graphLocation = TestHelper.convertToRelative(TinkerMemoryGraphTest.class,
- TestHelper.makeTestDataPath(TinkerMemoryGraphTest.class))
- + "shouldPersistToGryoRelative.kryo";
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo");
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateModern(graph);
- graph.close();
-
- final TinkerGraph reloadedGraph = TinkerGraph.open(conf);
- IoTest.assertModernGraph(reloadedGraph, true, false);
- reloadedGraph.close();
- }
-
- @Test
- public void shouldPersistToAnyGraphFormat() {
- final String graphLocation = TestHelper.makeTestDataFile(TinkerMemoryGraphTest.class, "shouldPersistToAnyGraphFormat.dat");
- final File f = new File(graphLocation);
- if (f.exists() && f.isFile()) f.delete();
-
- final Configuration conf = new BaseConfiguration();
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, TestIoBuilder.class.getName());
- conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, graphLocation);
- final TinkerGraph graph = TinkerGraph.open(conf);
- TinkerFactory.generateModern(graph);
-
- //Test write graph
- graph.close();
- assertEquals(TestIoBuilder.calledOnMapper, 1);
- assertEquals(TestIoBuilder.calledGraph, 1);
- assertEquals(TestIoBuilder.calledCreate, 1);
-
- try (BufferedOutputStream os = new BufferedOutputStream(new FileOutputStream(f))){
- os.write("dummy string".getBytes());
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- //Test read graph
- final TinkerGraph readGraph = TinkerGraph.open(conf);
- assertEquals(TestIoBuilder.calledOnMapper, 1);
- assertEquals(TestIoBuilder.calledGraph, 1);
- assertEquals(TestIoBuilder.calledCreate, 1);
- }
-
@Test
public void shouldSerializeWithColorClassResolverToTinkerGraph() throws Exception {
final Map<String,Color> colors = new HashMap<>();
@@ -1060,41 +915,6 @@ public class TinkerMemoryGraphTest {
}
}
- public static class TestIoBuilder implements Io.Builder {
-
- static int calledGraph, calledCreate, calledOnMapper;
-
- public TestIoBuilder(){
- //Looks awkward to reset static vars inside a constructor, but makes sense from testing perspective
- calledGraph = 0;
- calledCreate = 0;
- calledOnMapper = 0;
- }
-
- @Override
- public Io.Builder<? extends Io> onMapper(final Consumer onMapper) {
- calledOnMapper++;
- return this;
- }
-
- @Override
- public Io.Builder<? extends Io> graph(final Graph graph) {
- calledGraph++;
- return this;
- }
-
- @Override
- public Io create() {
- calledCreate++;
- return mock(Io.class);
- }
-
- @Override
- public boolean requiresVersion(final Object version) {
- return false;
- }
- }
-
@Test
public void shouldGroupMultiPropertyValuesUnderSingleKeyForPropertyMap() throws Exception {
try (final TinkerGraph graph = TinkerGraph.open()) {
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
new file mode 100644
index 0000000000..561320bcff
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.BaseConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Engine-agnostic conformance suite ("TCK") for pluggable {@link TinkerStorage} engines. A concrete engine is tested
+ * by subclassing this and returning the {@code gremlin.tinkergraph.storage} value that selects it (an engine name or a
+ * fully-qualified class name). Every test opens a {@link TinkerStorageGraph} backed by a fresh temporary directory,
+ * mutates it, reopens from the same configuration, and asserts the data survived. A new engine drops in by adding one
+ * subclass.
+ */
+public abstract class AbstractTinkerStorageConformanceTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private String location;
+
+ /**
+ * The {@code gremlin.tinkergraph.storage} configuration value that selects the engine under test.
+ */
+ protected abstract String storageEngine();
+
+ @Before
+ public void setUp() throws Exception {
+ location = tempFolder.newFolder("storage").getAbsolutePath();
+ }
+
+ protected Configuration config() {
+ final Configuration conf = new BaseConfiguration();
+ conf.setProperty(Graph.GRAPH, TinkerStorageGraph.class.getName());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE, storageEngine());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, location);
+ return conf;
+ }
+
+ protected TinkerStorageGraph open() {
+ return TinkerStorageGraph.open(config());
+ }
+
+ @Test
+ public void shouldPersistVerticesAndEdgesAcrossReopen() {
+ TinkerStorageGraph graph = open();
+ final Vertex marko = graph.addVertex(T.id, 1, T.label, "person", "name", "marko", "age", 29);
+ final Vertex lop = graph.addVertex(T.id, 3, T.label, "software", "name", "lop", "lang", "java");
+ marko.addEdge("created", lop, T.id, 9, "weight", 0.4);
+ graph.tx().commit();
+ graph.close();
+
+ graph = open();
+ assertEquals(2, countOf(graph.vertices()));
+ assertEquals(1, countOf(graph.edges()));
+ final Vertex reMarko = graph.vertices(1).next();
+ assertEquals("marko", reMarko.value("name"));
+ assertEquals(Integer.valueOf(29), reMarko.value("age"));
+ final Edge reCreated = graph.edges(9).next();
+ assertEquals("created", reCreated.label());
+ assertEquals(0.4, reCreated.<Double>value("weight"), 0.0001);
+ assertEquals(Integer.valueOf(1), reCreated.outVertex().id());
+ assertEquals(Integer.valueOf(3), reCreated.inVertex().id());
+ graph.close();
+ }
+
+ @Test
+ public void shouldPersistAcrossMultipleCommits() {
+ TinkerStorageGraph graph = open();
+ for (int i = 0; i < 10; i++) {
+ graph.addVertex(T.id, i, "value", i);
+ graph.tx().commit();
+ }
+ graph.close();
+
+ graph = open();
+ assertEquals(10, countOf(graph.vertices()));
+ for (int i = 0; i < 10; i++)
+ assertEquals(Integer.valueOf(i), graph.vertices(i).next().value("value"));
+ graph.close();
+ }
+
+ @Test
+ public void shouldPersistModificationsWithLastWriteWinning() {
+ TinkerStorageGraph graph = open();
+ final Vertex v = graph.addVertex(T.id, 1, "name", "original");
+ graph.tx().commit();
+ v.property("name", "updated");
+ graph.tx().commit();
+ graph.close();
+
+ graph = open();
+ assertEquals("updated", graph.vertices(1).next().value("name"));
+ graph.close();
+ }
+
+ @Test
+ public void shouldNotPersistRemovedElements() {
+ TinkerStorageGraph graph = open();
+ final Vertex a = graph.addVertex(T.id, 1);
+ final Vertex b = graph.addVertex(T.id, 2);
+ final Edge e = a.addEdge("knows", b, T.id, 10);
+ graph.tx().commit();
+ e.remove();
+ b.remove();
+ graph.tx().commit();
+ graph.close();
+
+ graph = open();
+ assertEquals(1, countOf(graph.vertices()));
+ assertEquals(0, countOf(graph.edges()));
+ assertNotNull(graph.vertices(1).next());
+ assertFalse(graph.vertices(2).hasNext());
+ graph.close();
+ }
+
+ @Test
+ public void shouldNotPersistRolledBackTransaction() {
+ TinkerStorageGraph graph = open();
+ graph.addVertex(T.id, 1, "name", "committed");
+ graph.tx().commit();
+ graph.addVertex(T.id, 2, "name", "rolledback");
+ graph.tx().rollback();
+ graph.close();
+
+ graph = open();
+ assertEquals(1, countOf(graph.vertices()));
+ assertNotNull(graph.vertices(1).next());
+ assertFalse(graph.vertices(2).hasNext());
+ graph.close();
+ }
+
+ @Test
+ public void shouldPersistMetaPropertiesAndMultiProperties() {
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name());
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ final Vertex v = graph.addVertex(T.id, 1);
+ final VertexProperty<String> vp = v.property(VertexProperty.Cardinality.list, "name", "marko");
+ vp.property("acl", "public");
+ v.property(VertexProperty.Cardinality.list, "name", "marko a. rodriguez");
+ graph.tx().commit();
+ graph.close();
+
+ graph = TinkerStorageGraph.open(conf);
+ final Vertex reV = graph.vertices(1).next();
+ assertEquals(2, countOf(reV.properties("name")));
+ final Iterator<VertexProperty<Object>> props = reV.properties("name");
+ boolean foundAcl = false;
+ while (props.hasNext()) {
+ final VertexProperty<Object> p = props.next();
+ if (p.properties("acl").hasNext()) {
+ assertEquals("public", p.properties("acl").next().value());
+ foundAcl = true;
+ }
+ }
+ assertTrue("meta-property should survive persistence", foundAcl);
+ graph.close();
+ }
+
+ @Test
+ public void shouldPreserveStateAfterCompact() {
+ TinkerStorageGraph graph = open();
+ for (int i = 0; i < 5; i++) {
+ graph.addVertex(T.id, i, "value", i);
+ graph.tx().commit();
+ }
+ graph.compact();
+ // keep writing after compaction to exercise the truncated log
+ graph.addVertex(T.id, 100, "value", 100);
+ graph.tx().commit();
+ graph.close();
+
+ graph = open();
+ assertEquals(6, countOf(graph.vertices()));
+ assertEquals(Integer.valueOf(100), graph.vertices(100).next().value("value"));
+ assertEquals(Integer.valueOf(3), graph.vertices(3).next().value("value"));
+ graph.close();
+ }
+
+ @Test
+ public void shouldReopenEmptyGraph() {
+ TinkerStorageGraph graph = open();
+ graph.close();
+
+ graph = open();
+ assertEquals(0, countOf(graph.vertices()));
+ assertEquals(0, countOf(graph.edges()));
+ graph.close();
+ }
+
+ @Test
+ public void shouldReportPersistenceFeature() {
+ final TinkerStorageGraph graph = open();
+ assertTrue(graph.features().graph().supportsPersistence());
+ graph.close();
+ }
+
+ private static long countOf(final Iterator<?> it) {
+ long count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count++;
+ }
+ return count;
+ }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
new file mode 100644
index 0000000000..43c53d66f0
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.RandomAccessFile;
+import java.util.Iterator;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Runs the shared {@link AbstractTinkerStorageConformanceTest} suite against the {@link GraphBinaryStorage} engine and
+ * adds engine-specific tests for the on-disk log layout.
+ */
+public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest {
+
+ @Override
+ protected String storageEngine() {
+ return "graphbinary";
+ }
+
+ @Test
+ public void shouldWriteSnapshotAndLogFiles() {
+ final TinkerStorageGraph graph = open();
+ graph.addVertex(T.id, 1);
+ graph.tx().commit();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ assertTrue(new File(location, GraphBinaryStorage.LOG_FILE).exists());
+ graph.close();
+ // close compacts, producing a snapshot and truncating the log
+ assertTrue(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).exists());
+ }
+
+ @Test
+ public void shouldRecoverFromTruncatedTrailingFrame() throws Exception {
+ TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ graph.addVertex(T.id, 1, "value", 1);
+ graph.tx().commit();
+ graph.addVertex(T.id, 2, "value", 2);
+ graph.tx().commit();
+ // do NOT close (avoid compaction) so the raw log is preserved
+ graph.tx().close();
+
+ // simulate a crash mid-append by appending a partial (garbage) trailing frame to the log
+ final File logFile = new File(location, GraphBinaryStorage.LOG_FILE);
+ try (final RandomAccessFile raf = new RandomAccessFile(logFile, "rw")) {
+ raf.seek(raf.length());
+ // a length prefix promising 100 bytes, but only a couple follow — a torn write
+ raf.writeInt(100);
+ raf.write(new byte[]{ 0x01, 0x02 });
+ }
+
+ // reopening must recover the two fully-committed vertices and ignore the torn frame
+ graph = open();
+ assertEquals(2, countOf(graph.vertices()));
+ assertEquals(Integer.valueOf(1), graph.vertices(1).next().value("value"));
+ assertEquals(Integer.valueOf(2), graph.vertices(2).next().value("value"));
+ graph.close();
+ }
+
+ private static long countOf(final Iterator<?> it) {
+ long count = 0;
+ while (it.hasNext()) {
+ it.next();
+ count++;
+ }
+ return count;
+ }
+}
(tinkerpop) 06/20: Auto-compact TinkerStorageGraph log to bound its growth
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 78474a129d05478dbe27822931adac562dc6adc1
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 15:45:06 2026 +0000
Auto-compact TinkerStorageGraph log to bound its growth
A long-running graph that is never explicitly closed grew its append log
without bound, and its restart replay cost grew with it. The GraphBinary
engine now tracks appended log bytes and folds the log into a snapshot on
commit once it exceeds a threshold, configurable via
gremlin.tinkergraph.storage.compactThreshold (default 64MB, 0 to disable).
A new no-op-by-default TinkerStorage.maybeCompact SPI hook drives this, so
existing engines are unaffected. Compaction runs inline under the commit lock.
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 2 +-
.../gremlin/tinkergraph/structure/TinkerGraph.java | 8 ++++
.../tinkergraph/structure/TinkerTransaction.java | 3 ++
.../structure/storage/GraphBinaryStorage.java | 23 ++++++++++
.../structure/storage/TinkerStorage.java | 12 +++++
.../structure/storage/GraphBinaryStorageTest.java | 51 ++++++++++++++++++++++
6 files changed, 98 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index b67d2ca09e..6a6b2bd684 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,7 +28,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message.
* Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)*
* Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)*
-* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast.
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast; the append log auto-compacts once it exceeds `gremlin.tinkergraph.storage.compactThresh [...]
* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
[[release-4-0-0-beta-3]]
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 3bb5200bef..65a02537c6 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -67,6 +67,14 @@ public interface TinkerGraph extends Graph {
* {@code org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.SyncMode}.
*/
String GREMLIN_TINKERGRAPH_STORAGE_SYNC = "gremlin.tinkergraph.storage.sync";
+ /**
+ * The size in bytes at which a {@link TinkerStorageGraph} storage engine automatically compacts its append log on
+ * commit, bounding the log growth (and restart replay cost) of a long-running graph that is never explicitly
+ * closed. Defaults to 67108864 (64 MB). Set to {@code 0} to disable automatic compaction and rely on
+ * {@code close()} or an explicit {@code compact()}. Only meaningful when {@link #GREMLIN_TINKERGRAPH_STORAGE} is
+ * set.
+ */
+ String GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD = "gremlin.tinkergraph.storage.compactThreshold";
String GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES = "gremlin.tinkergraph.allowNullPropertyValues";
String GREMLIN_TINKERGRAPH_SERVICE = "gremlin.tinkergraph.service";
String GREMLIN_TINKERGRAPH_VERTEX_LABEL_CARDINALITY = "gremlin.tinkergraph.vertexLabelCardinality";
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
index 50c48bde80..36d5167c13 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
@@ -186,6 +186,9 @@ final class TinkerTransaction extends AbstractThreadLocalTransaction {
try {
graph.storage.persist(txVersion, toVertexMutations(changedVertices), toEdgeMutations(changedEdges));
graph.storage.flush();
+ // bound log growth for a long-running graph that is never explicitly closed; no-op unless the
+ // engine's accumulated log has crossed its threshold
+ graph.storage.maybeCompact(graph);
} finally {
graph.storageCommitLock.unlock();
}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 9b31349a2a..c9eecba9e5 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -91,9 +91,16 @@ public final class GraphBinaryStorage implements TinkerStorage {
private File snapshotFile;
private File logFile;
+ /**
+ * Default automatic-compaction threshold: 64 MB of appended log since the last compaction.
+ */
+ static final long DEFAULT_COMPACT_THRESHOLD_BYTES = 64L * 1024 * 1024;
+
private DataOutputStream logOut;
private FileOutputStream logFos;
private SyncMode syncMode = SyncMode.COMMIT;
+ private long compactThresholdBytes = DEFAULT_COMPACT_THRESHOLD_BYTES;
+ private long logBytesSinceCompaction = 0;
private boolean closed = false;
@Override
@@ -106,6 +113,10 @@ public final class GraphBinaryStorage implements TinkerStorage {
this.snapshotFile = new File(directory, SNAPSHOT_FILE);
this.logFile = new File(directory, LOG_FILE);
this.syncMode = SyncMode.fromConfigValue(config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, null));
+ this.compactThresholdBytes = config.getLong(
+ TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, DEFAULT_COMPACT_THRESHOLD_BYTES);
+ // seed the counter with any pre-existing log so a graph reopened with a large log still compacts promptly
+ this.logBytesSinceCompaction = logFile.exists() ? logFile.length() : 0;
ensureDirectory();
}
@@ -210,6 +221,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
try {
final byte[] frame = encodeRecord(txVersion, changedVertices, changedEdges);
writeFrame(logOut, frame);
+ logBytesSinceCompaction += Integer.BYTES + frame.length; // length prefix + payload
} catch (IOException ex) {
throw new UncheckedIOException("Could not append transaction to storage log", ex);
}
@@ -304,6 +316,17 @@ public final class GraphBinaryStorage implements TinkerStorage {
} catch (IOException ex) {
throw new UncheckedIOException("Could not finalize storage snapshot", ex);
}
+
+ // the log is now empty; the accumulated state lives in the snapshot
+ logBytesSinceCompaction = 0;
+ }
+
+ @Override
+ public void maybeCompact(final AbstractTinkerGraph graph) {
+ if (closed || compactThresholdBytes <= 0)
+ return;
+ if (logBytesSinceCompaction >= compactThresholdBytes)
+ compact(graph);
}
/**
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
index 49dcedb1ba..ab4745dcf2 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/TinkerStorage.java
@@ -90,6 +90,18 @@ public interface TinkerStorage extends AutoCloseable {
*/
void compact(AbstractTinkerGraph graph);
+ /**
+ * Compact if the engine's accumulated data has grown past its own threshold, otherwise do nothing. Called on the
+ * commit path after {@link #flush()} (while the graph's commit lock is held) so a long-running graph that is never
+ * explicitly closed does not grow its backing storage without bound. The default is a no-op, leaving compaction
+ * entirely under the control of {@link #compact(AbstractTinkerGraph)} and {@link #close()}.
+ *
+ * @param graph the graph whose current committed state would be snapshotted
+ */
+ default void maybeCompact(final AbstractTinkerGraph graph) {
+ // no-op by default; engines that accumulate an on-disk log override this to bound its growth
+ }
+
/**
* {@inheritDoc}
* <p/>
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 05e9970504..8113e03d18 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -105,6 +105,57 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
graph.close();
}
+ @Test
+ public void shouldAutoCompactWhenLogExceedsThreshold() {
+ // a small threshold makes automatic compaction fire mid-run, without any explicit compact()/close()
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 2048L);
+ final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ try {
+ for (int i = 0; i < 200; i++) {
+ graph.addVertex(T.id, i, "value", i, "pad", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+ graph.tx().commit();
+ }
+ // auto-compaction should have folded the log into a snapshot and truncated it well below the total
+ // bytes written, so the live log stays bounded rather than growing with every commit
+ final File logFile = new File(location, GraphBinaryStorage.LOG_FILE);
+ final File snapshotFile = new File(location, GraphBinaryStorage.SNAPSHOT_FILE);
+ assertTrue("expected a snapshot from auto-compaction", snapshotFile.exists());
+ assertTrue("expected the live log to stay bounded, was " + logFile.length(),
+ logFile.length() < 8192);
+ } finally {
+ graph.close();
+ }
+
+ // data must survive across reopen despite the mid-run compactions
+ graph = TinkerStorageGraph.open(conf);
+ try {
+ assertEquals(200, countOf(graph.vertices()));
+ assertEquals(Integer.valueOf(199), graph.vertices(199).next().value("value"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldNotAutoCompactWhenThresholdIsZero() {
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, 0L);
+ final String location = conf.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ final TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ try {
+ for (int i = 0; i < 50; i++) {
+ graph.addVertex(T.id, i, "pad", "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+ graph.tx().commit();
+ }
+ // with auto-compaction disabled, no snapshot appears until close()/compact()
+ assertTrue(!new File(location, GraphBinaryStorage.SNAPSHOT_FILE).exists());
+ } finally {
+ graph.close();
+ }
+ }
+
@Test
public void shouldRecoverFromTruncatedTrailingFrame() throws Exception {
TinkerStorageGraph graph = open();
(tinkerpop) 20/20: Document TinkerStorageGraph persistence and fix stale class doc
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 9ebb314541764615d752fe7d94fa865cde3afb0d
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Thu Aug 20 18:26:32 2026 +0000
Document TinkerStorageGraph persistence and fix stale class doc
Rewrite the TinkerStorageGraph class Javadoc, which still described disk
storage as planned future work. Add a commented, persistence-enabled server
sample (tinkerstoragegraph-persistent.properties) and a Gremlin Server
subsection in the persistence reference. Rename the misnamed console
tinkergraph-gryo.properties to tinkergraph-storage.properties and comment the
credentials sample.
Assisted-by: Claude Code:claude-opus-4-8
---
.../reference/implementations-tinkergraph.asciidoc | 18 ++++++++++
...o.properties => tinkergraph-storage.properties} | 4 +++
.../conf/tinkergraph-credentials.properties | 3 ++
.../conf/tinkerstoragegraph-persistent.properties | 40 ++++++++++++++++++++++
.../tinkergraph/structure/TinkerStorageGraph.java | 14 ++++++--
5 files changed, 76 insertions(+), 3 deletions(-)
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index 7f7db73fbe..6b3a8d086c 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -471,6 +471,24 @@ the storage format version it was written with. Opening a store written in a for
with a clear error rather than misreading the data. There is no in-place format migration. To move a graph across an
incompatible storage format, export it with the `io()` step before upgrading and read it back afterward.
+Gremlin Server persists a graph the same way. A graph in the server's `graphs` configuration is pointed at a
+properties file that sets the storage keys, and the server opens that graph at startup, commits to it as clients
+request, and closes it on shutdown. A graph configured without a storage engine remains transactional but in-memory,
+so its data is lost when the server stops.
+
+[source,properties]
+----
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph
+gremlin.tinkergraph.storage=graphbinary
+gremlin.tinkergraph.graphLocation=/data/mygraph
+----
+
+[source,yaml]
+----
+graphs: {
+ graph: conf/tinkerstoragegraph-persistent.properties }
+----
+
Persistence is distinct from interchange. `TinkerMemoryGraph` is purely in-memory and does not persist. To move data in
or out of any TinkerGraph in an interchange format such as GraphML, GraphSON, or Gryo, use the `io()` step directly:
diff --git a/gremlin-console/conf/tinkergraph-gryo.properties b/gremlin-console/conf/tinkergraph-storage.properties
similarity index 69%
rename from gremlin-console/conf/tinkergraph-gryo.properties
rename to gremlin-console/conf/tinkergraph-storage.properties
index 312dc3f9b6..2a04bb7d37 100644
--- a/gremlin-console/conf/tinkergraph-gryo.properties
+++ b/gremlin-console/conf/tinkergraph-storage.properties
@@ -15,7 +15,11 @@
# specific language governing permissions and limitations
# under the License.
+# Sample configuration for a durable, transactional TinkerStorageGraph. Opening a graph with this configuration
+# (for example via GraphFactory) produces a TinkerStorageGraph that persists committed transactions to graphLocation.
gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph
+# built-in storage engine; without this key the graph is transactional but in-memory only
gremlin.tinkergraph.storage=graphbinary
+# directory holding the durable data (created if absent; a location may be opened by only one graph at a time)
gremlin.tinkergraph.graphLocation=/tmp/tinkergraph
diff --git a/gremlin-server/conf/tinkergraph-credentials.properties b/gremlin-server/conf/tinkergraph-credentials.properties
index 4597fabcd3..fb4225b376 100644
--- a/gremlin-server/conf/tinkergraph-credentials.properties
+++ b/gremlin-server/conf/tinkergraph-credentials.properties
@@ -16,5 +16,8 @@
# under the License.
gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph
gremlin.tinkergraph.vertexIdManager=LONG
+# This credential store is an in-memory TinkerGraph. TinkerGraph no longer auto-loads from disk on open, so
+# SimpleAuthenticator reads the store explicitly from graphLocation using graphFormat at startup. (A durable
+# TinkerStorageGraph, by contrast, manages its own persistence and needs no graphFormat.)
gremlin.tinkergraph.graphLocation=data/credentials.kryo
gremlin.tinkergraph.graphFormat=gryo
\ No newline at end of file
diff --git a/gremlin-server/conf/tinkerstoragegraph-persistent.properties b/gremlin-server/conf/tinkerstoragegraph-persistent.properties
new file mode 100644
index 0000000000..dac2460298
--- /dev/null
+++ b/gremlin-server/conf/tinkerstoragegraph-persistent.properties
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Sample configuration for a durable, transactional TinkerStorageGraph on Gremlin Server. Reference it from the
+# server's "graphs" block, for example in gremlin-server-transaction.yaml:
+# graphs: { graph: conf/tinkerstoragegraph-persistent.properties }
+gremlin.graph=org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph
+
+# Enables durable persistence. "graphbinary" is the built-in storage engine; the fully qualified class name of a
+# custom TinkerStorage implementation may be used instead. Without this key the graph is transactional but in-memory.
+gremlin.tinkergraph.storage=graphbinary
+# Directory holding the durable data. Required when a storage engine is set. It is created if absent and may be
+# opened by only one graph at a time (single writer, whether in this JVM or another process).
+gremlin.tinkergraph.graphLocation=/tmp/tinkerstoragegraph
+
+gremlin.tinkergraph.vertexIdManager=LONG
+gremlin.tinkergraph.edgeIdManager=LONG
+gremlin.tinkergraph.vertexPropertyIdManager=LONG
+
+# Optional storage tuning (defaults shown, uncomment to change):
+# "commit" forces each transaction to disk (survives OS crash/power loss); "os" is faster but survives only a JVM crash
+#gremlin.tinkergraph.storage.sync=commit
+# auto-compact the append log once it grows past this many bytes (default 67108864 = 64MB; 0 disables)
+#gremlin.tinkergraph.storage.compactThreshold=67108864
+# persist auto-generated vertex-property ids across a reopen (default false)
+#gremlin.tinkergraph.storage.preserveVertexPropertyIds=false
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
index 803cc27dcf..ae9fcb2b48 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
@@ -49,9 +49,17 @@ import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
/**
- * The transactional implementation of the {@link TinkerGraph} interface, in-memory with optional persistence on
- * calls to {@link #close()}. It is planned that this implementation will optionally support simple storage to disk
- * built on its transaction functionality.
+ * The transactional implementation of the {@link TinkerGraph} interface. It provides {@code read committed}
+ * transaction isolation with optimistic locking and, when a storage engine is configured, durable persistence to
+ * disk. With no storage engine configured it is an in-memory transactional graph that retains nothing across
+ * restarts.
+ * <p/>
+ * Persistence is pluggable through the {@link org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.TinkerStorage}
+ * SPI and enabled with the {@code gremlin.tinkergraph.storage} and {@code gremlin.tinkergraph.graphLocation}
+ * configuration keys. Each committed transaction is written through to the storage engine before the in-memory commit
+ * is applied, and reopening the same location replays the persisted commits to rebuild the graph. A storage location
+ * is single-writer: it is guarded by an exclusive {@link org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.DirectoryLock}
+ * so a second open of the same directory fails rather than corrupting the data.
*
* @author Valentyn Kahamlyk
*/
(tinkerpop) 17/20: Test TinkerStorage durability, lifecycle, and dictionary crash window
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit a90ef636c36610c531ceab614409146704eb3693
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 16:34:36 2026 +0000
Test TinkerStorage durability, lifecycle, and dictionary crash window
Add TCK cases for repeated compaction cycles, multiple open/close sessions,
and concurrent commits with distinct keys. Add a crash-consistency test for the
compaction crash window where a dead dictionary key forces the surviving log to
diverge from the new snapshot's dictionary — guarding the decision to preserve
dictionary numbering across compaction rather than renumber.
Assisted-by: Claude Code:claude-opus-4-8
---
.../AbstractTinkerStorageConformanceTest.java | 112 +++++++++++++++++++++
.../storage/StorageCrashConsistencyTest.java | 43 ++++++++
2 files changed, 155 insertions(+)
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
index f5ec5033f4..34afede360 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
@@ -490,6 +490,118 @@ public abstract class AbstractTinkerStorageConformanceTest {
}
}
+ @Test
+ public void shouldSurviveRepeatedCompactionCycles() {
+ TinkerStorageGraph graph = open();
+ try {
+ graph.addVertex(T.id, 1, "a", 1);
+ graph.tx().commit();
+ graph.compact();
+ graph.addVertex(T.id, 2, "b", 2);
+ graph.tx().commit();
+ graph.compact();
+ graph.addVertex(T.id, 3, "c", 3);
+ graph.tx().commit();
+ graph.compact();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ assertEquals(3, countOf(graph.vertices()));
+ assertEquals(Integer.valueOf(1), graph.vertices(1).next().value("a"));
+ assertEquals(Integer.valueOf(2), graph.vertices(2).next().value("b"));
+ assertEquals(Integer.valueOf(3), graph.vertices(3).next().value("c"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldSurviveMultipleOpenCloseSessions() {
+ TinkerStorageGraph graph = open();
+ try {
+ graph.addVertex(T.id, 1, "n", "one");
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ graph.addVertex(T.id, 2, "n", "two");
+ graph.tx().commit();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ graph.addVertex(T.id, 3, "n", "three");
+ graph.tx().commit();
+ graph.compact();
+ } finally {
+ graph.close();
+ }
+ graph = open();
+ try {
+ assertEquals(3, countOf(graph.vertices()));
+ assertEquals("one", graph.vertices(1).next().value("n"));
+ assertEquals("two", graph.vertices(2).next().value("n"));
+ assertEquals("three", graph.vertices(3).next().value("n"));
+ } finally {
+ graph.close();
+ }
+ }
+
+ @Test
+ public void shouldRoundTripConcurrentCommitsWithDistinctKeys() throws Exception {
+ // concurrent commits that each introduce a distinct property key stress dictionary growth under the
+ // commit-write lock; on reopen every distinct key must resolve
+ final int threads = 8;
+ final int perThread = 25;
+ final TinkerStorageGraph writeGraph = open();
+ try {
+ final ExecutorService pool = Executors.newFixedThreadPool(threads);
+ final CountDownLatch start = new CountDownLatch(1);
+ final List<Future<?>> futures = new ArrayList<>();
+ for (int t = 0; t < threads; t++) {
+ final int threadId = t;
+ futures.add(pool.submit(() -> {
+ start.await();
+ for (int i = 0; i < perThread; i++) {
+ final int id = threadId * perThread + i;
+ writeGraph.addVertex(T.id, id, "k_" + threadId + "_" + i, id);
+ writeGraph.tx().commit();
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (final Future<?> f : futures)
+ f.get(60, TimeUnit.SECONDS);
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS));
+ } finally {
+ writeGraph.close();
+ }
+ final TinkerStorageGraph reopened = open();
+ try {
+ final int expected = threads * perThread;
+ assertEquals(expected, countOf(reopened.vertices()));
+ for (int t = 0; t < threads; t++) {
+ for (int i = 0; i < perThread; i++) {
+ final int id = threadId(t, i, perThread);
+ assertEquals(Integer.valueOf(id), reopened.vertices(id).next().value("k_" + t + "_" + i));
+ }
+ }
+ } finally {
+ reopened.close();
+ }
+ }
+
+ private static int threadId(final int t, final int i, final int perThread) {
+ return t * perThread + i;
+ }
+
private static long countOf(final Iterator<?> it) {
long count = 0;
while (it.hasNext()) {
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
index 8f3c475f13..4bbb3854ce 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCrashConsistencyTest.java
@@ -22,6 +22,7 @@ import org.apache.commons.configuration2.BaseConfiguration;
import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
import org.junit.Before;
@@ -169,6 +170,48 @@ public class StorageCrashConsistencyTest {
assertReopensTo(1, 2);
}
+ @Test
+ public void shouldRecoverCrashWindowWithADeadKeyForcingDictionaryDivergence() throws Exception {
+ // Guards the preserve-dictionary-numbering decision. A key ("alpha") is deleted from the live graph after it
+ // is in the dictionary, then a new key ("gamma") is added. Preserving numbering keeps alpha's id forever, so
+ // the surviving log's gamma ref still matches the new snapshot's dictionary. Renumbering on compaction would
+ // instead drop the now-dead alpha and shift gamma to a lower id, so the log's higher-numbered gamma ref would
+ // no longer resolve. A single-key (or no-delete) state cannot tell the two apart.
+ final TinkerStorageGraph g = open();
+ g.addVertex(T.id, 1, "alpha", 1);
+ g.tx().commit();
+ g.addVertex(T.id, 2, "beta", 2);
+ g.tx().commit();
+ g.compact(); // snapshot holds alpha and beta in the dictionary at stable ids
+
+ g.vertices(1).next().remove(); // alpha becomes a dead key: retained only under preserve-numbering
+ g.tx().commit();
+ // re-write the surviving vertex: its record now carries a bare reference to the pre-existing key "beta"
+ // (not re-appended) plus a new key "gamma". If compaction renumbered, "beta"'s id would shift and this bare
+ // reference would resolve to the wrong key on replay.
+ g.vertices(2).next().property("gamma", "g");
+ g.tx().commit();
+ final byte[] logNotYetDeleted = Files.readAllBytes(logFile.toPath());
+ g.compact(); // new full-dictionary snapshot (preserved numbering), then log truncated
+ final byte[] newSnapshot = Files.readAllBytes(snapshotFile.toPath());
+ g.close();
+
+ // reconstruct the crash window: new snapshot in place, old log not yet deleted
+ Files.write(snapshotFile.toPath(), newSnapshot);
+ Files.write(logFile.toPath(), logNotYetDeleted);
+
+ final TinkerStorageGraph reopened = open();
+ try {
+ assertEquals(1, countOf(reopened.vertices())); // only the surviving vertex 2
+ assertEquals(0, countOf(reopened.vertices(1))); // alpha's vertex was deleted
+ final Vertex v = reopened.vertices(2).next();
+ assertEquals(Integer.valueOf(2), v.value("beta"));
+ assertEquals("g", v.value("gamma"));
+ } finally {
+ reopened.close();
+ }
+ }
+
private static long countOf(final Iterator<?> it) {
long count = 0;
while (it.hasNext()) {
(tinkerpop) 03/20: Make TinkerStorageGraph commits fsync-durable with a configurable sync mode
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 2a258ffba8935246336e1506ca366914d21e49bc
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Mon Aug 3 20:03:06 2026 +0000
Make TinkerStorageGraph commits fsync-durable with a configurable sync mode
The GraphBinary storage engine's flush() only pushed bytes into the OS page
cache, so an acknowledged commit could be lost on OS crash or power loss.
flush() now fsyncs on commit, and compaction is made crash-safe: the snapshot
is fsync'd and atomically renamed into place with directory fsyncs before the
log is truncated.
A new gremlin.tinkergraph.storage.sync config key selects the durability mode:
'commit' (default, fsync every commit) or 'os' (flush to the OS only; survives
process crash but not power loss).
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 2 +-
.../gremlin/tinkergraph/structure/TinkerGraph.java | 8 +++
.../structure/storage/GraphBinaryStorage.java | 80 +++++++++++++++++++---
.../tinkergraph/structure/storage/SyncMode.java | 70 +++++++++++++++++++
.../structure/storage/GraphBinaryStorageTest.java | 51 ++++++++++++++
.../structure/storage/SyncModeTest.java | 44 ++++++++++++
6 files changed, 244 insertions(+), 11 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 0f7480173a..5757e51ff8 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,7 +28,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message.
* Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)*
* Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)*
-* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine.
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability.
* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
[[release-4-0-0-beta-3]]
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 0935c38fdb..3bb5200bef 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -59,6 +59,14 @@ public interface TinkerGraph extends Graph {
* the graph holds data only in memory. Not valid on {@link TinkerMemoryGraph}.
*/
String GREMLIN_TINKERGRAPH_STORAGE = "gremlin.tinkergraph.storage";
+ /**
+ * The durability mode a {@link TinkerStorageGraph} storage engine applies on commit. Either {@code commit}
+ * (default) to {@code fsync} every commit so acknowledged commits survive an OS crash or power loss, or {@code os}
+ * to only flush to the operating system so commits survive a JVM process crash but may be lost on OS crash or
+ * power loss. Only meaningful when {@link #GREMLIN_TINKERGRAPH_STORAGE} is set. See
+ * {@code org.apache.tinkerpop.gremlin.tinkergraph.structure.storage.SyncMode}.
+ */
+ String GREMLIN_TINKERGRAPH_STORAGE_SYNC = "gremlin.tinkergraph.storage.sync";
String GREMLIN_TINKERGRAPH_ALLOW_NULL_PROPERTY_VALUES = "gremlin.tinkergraph.allowNullPropertyValues";
String GREMLIN_TINKERGRAPH_SERVICE = "gremlin.tinkergraph.service";
String GREMLIN_TINKERGRAPH_VERTEX_LABEL_CARDINALITY = "gremlin.tinkergraph.vertexLabelCardinality";
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index b07891a93c..9b31349a2a 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -44,6 +44,11 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UncheckedIOException;
+import java.nio.channels.FileChannel;
+import java.nio.file.AtomicMoveNotSupportedException;
+import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
@@ -58,6 +63,10 @@ import java.util.Map;
* folded result re-applied to the in-memory graph. {@link #compact(AbstractTinkerGraph)} rewrites the snapshot from the
* current committed state and truncates the log.
* <p/>
+ * On commit the appended record is made durable according to the configured {@link SyncMode}: {@link SyncMode#COMMIT}
+ * (default) {@code fsync}s so the commit survives an OS crash or power loss, while {@link SyncMode#OS} only flushes to
+ * the operating system.
+ * <p/>
* The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
*/
public final class GraphBinaryStorage implements TinkerStorage {
@@ -83,6 +92,8 @@ public final class GraphBinaryStorage implements TinkerStorage {
private File logFile;
private DataOutputStream logOut;
+ private FileOutputStream logFos;
+ private SyncMode syncMode = SyncMode.COMMIT;
private boolean closed = false;
@Override
@@ -94,6 +105,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
this.directory = new File(location);
this.snapshotFile = new File(directory, SNAPSHOT_FILE);
this.logFile = new File(directory, LOG_FILE);
+ this.syncMode = SyncMode.fromConfigValue(config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, null));
ensureDirectory();
}
@@ -242,7 +254,12 @@ public final class GraphBinaryStorage implements TinkerStorage {
return;
if (logOut != null) {
try {
+ // flush the JVM buffer into the OS page cache; durable against a JVM process crash
logOut.flush();
+ // in COMMIT mode also force the OS page cache to the device, so an acknowledged commit is durable
+ // against an OS crash or power loss. OS mode stops at the flush above and accepts that weaker guarantee.
+ if (syncMode == SyncMode.COMMIT)
+ logFos.getFD().sync();
} catch (IOException ex) {
throw new UncheckedIOException("Could not flush storage log", ex);
}
@@ -253,27 +270,67 @@ public final class GraphBinaryStorage implements TinkerStorage {
public void compact(final AbstractTinkerGraph graph) {
if (closed)
return;
- // Write a fresh snapshot of the current committed state, then truncate the log.
+ // Write a fresh snapshot of the current committed state, then truncate the log. This must be crash-safe: at
+ // no point may a crash leave the store without a readable snapshot-or-log covering the committed state.
+ // Ordering is write-tmp -> fsync tmp -> atomically rename tmp over the snapshot -> fsync dir (the rename is
+ // now durable) -> delete the log -> fsync dir. The old snapshot is only ever replaced by an atomic rename, so
+ // a crash at any step leaves either the old (snapshot + log) or the new (snapshot) intact — never neither.
closeLog();
ensureDirectory();
final File tmp = new File(directory, SNAPSHOT_FILE + ".tmp");
- try (final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(tmp)))) {
+ try (final FileOutputStream fos = new FileOutputStream(tmp);
+ final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(fos))) {
final byte[] frame = encodeSnapshot(graph);
if (frame.length > 0)
writeFrame(out, frame);
out.flush();
+ // force the snapshot's bytes to the device before it is renamed into place
+ fos.getFD().sync();
} catch (IOException ex) {
throw new UncheckedIOException("Could not write storage snapshot", ex);
}
- if (snapshotFile.exists() && !snapshotFile.delete())
- throw new UncheckedIOException(new IOException("Could not replace snapshot " + snapshotFile));
- if (!tmp.renameTo(snapshotFile))
- throw new UncheckedIOException(new IOException("Could not rename snapshot into place " + snapshotFile));
+ try {
+ // atomically replace the snapshot; no delete-then-rename window where the snapshot is briefly absent
+ atomicMove(tmp, snapshotFile);
+ // fsync the directory so the rename survives a crash before we touch the log
+ syncDirectory();
+
+ // truncate the log now that the snapshot durably reflects the committed state
+ if (logFile.exists() && !logFile.delete())
+ throw new IOException("Could not truncate storage log " + logFile);
+ // fsync the directory again so the log's removal is durable
+ syncDirectory();
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Could not finalize storage snapshot", ex);
+ }
+ }
- // truncate the log
- if (logFile.exists() && !logFile.delete())
- throw new UncheckedIOException(new IOException("Could not truncate storage log " + logFile));
+ /**
+ * Atomically move {@code source} onto {@code target}, replacing any existing target. Falls back to a non-atomic
+ * replacing move on filesystems that do not support atomic moves.
+ */
+ private static void atomicMove(final File source, final File target) throws IOException {
+ try {
+ Files.move(source.toPath(), target.toPath(),
+ StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING);
+ } catch (AtomicMoveNotSupportedException anse) {
+ Files.move(source.toPath(), target.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+
+ /**
+ * fsync the storage directory so that recent namespace changes (a rename into place, a file deletion) are durable.
+ * A directory fsync is required because those operations only update the directory entry, which the earlier file
+ * fsync does not cover.
+ */
+ private void syncDirectory() {
+ try (final FileChannel dirChannel = FileChannel.open(directory.toPath(), StandardOpenOption.READ)) {
+ dirChannel.force(true);
+ } catch (IOException ex) {
+ // some platforms (notably Windows) cannot open a directory as a channel; the atomic rename is the
+ // durability guarantee there, so treat inability to sync the directory as non-fatal
+ }
}
/**
@@ -316,7 +373,9 @@ public final class GraphBinaryStorage implements TinkerStorage {
private void ensureLogOpen() {
if (logOut == null) {
try {
- logOut = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(logFile, true)));
+ // retain the FileOutputStream so flush() can reach its FileDescriptor for fsync
+ logFos = new FileOutputStream(logFile, true);
+ logOut = new DataOutputStream(new BufferedOutputStream(logFos));
} catch (IOException ex) {
throw new UncheckedIOException("Could not open storage log for append", ex);
}
@@ -332,6 +391,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
throw new UncheckedIOException("Could not close storage log", ex);
} finally {
logOut = null;
+ logFos = null;
}
}
}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
new file mode 100644
index 0000000000..de049797fe
--- /dev/null
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+/**
+ * The durability mode a {@link TinkerStorage} engine applies when flushing a committed transaction to disk, selected
+ * with the {@code gremlin.tinkergraph.storage.sync} configuration key. Each value is a complete, mutually-exclusive
+ * choice; they are ordered from strongest to weakest durability.
+ * <p/>
+ * The name of a mode describes <em>when</em> data is made durable, not <em>what</em> action is taken: {@link #COMMIT}
+ * performs an {@code fsync} so acknowledged commits survive an OS crash or power loss, whereas {@link #OS} only pushes
+ * bytes into the operating system's page cache, so commits survive a crash of the JVM process but not of the OS.
+ */
+public enum SyncMode {
+
+ /**
+ * {@code fsync} on every commit. An acknowledged commit is durable against process crash, OS crash, and power
+ * loss. This is the default and the mode that honors the "each committed transaction is durably written to disk"
+ * contract.
+ */
+ COMMIT,
+
+ /**
+ * Flush to the operating system on every commit, but do not {@code fsync}. An acknowledged commit survives a crash
+ * of the JVM process but may be lost on an OS crash or power loss, since the data can still be sitting in the OS
+ * page cache. Faster than {@link #COMMIT}; use only when that weaker guarantee is acceptable.
+ */
+ OS;
+
+ // TODO: add an INTERVAL mode (group commit) — a peer value on this same key, encoded as "interval:<ms>", that
+ // fsyncs at most once per <ms> window rather than once per commit, bounding the crash-loss window by time while
+ // amortizing fsync cost across commits. It implies fsync (a batched COMMIT), so it slots in as a third
+ // mutually-exclusive mode without changing the meaning of COMMIT or OS. Deferred until concurrent commits are
+ // serialized, since group commit's payoff is amortizing one fsync across many concurrent committers.
+
+ /**
+ * Resolve a configuration value to a {@link SyncMode}, matched case-insensitively, defaulting to {@link #COMMIT}
+ * when unset.
+ *
+ * @param configValue the raw configuration value, or {@code null} when unset
+ * @return the resolved mode
+ * @throws IllegalArgumentException if the value does not name a known mode
+ */
+ public static SyncMode fromConfigValue(final String configValue) {
+ if (null == configValue)
+ return COMMIT;
+ try {
+ return SyncMode.valueOf(configValue.trim().toUpperCase());
+ } catch (IllegalArgumentException iae) {
+ throw new IllegalArgumentException(String.format(
+ "Unknown storage sync mode '%s'; valid values are 'commit' and 'os'", configValue), iae);
+ }
+ }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 43c53d66f0..4bdf5b8888 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+import org.apache.commons.configuration2.Configuration;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
@@ -53,6 +54,56 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
assertTrue(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).exists());
}
+ @Test
+ public void shouldPersistWithOsSyncMode() {
+ // 'os' is a weaker durability mode (no fsync); a graceful close/reopen must still round-trip the data. The
+ // OS-crash-loss window that distinguishes it from 'commit' cannot be exercised in a unit test.
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, "os");
+ TinkerStorageGraph graph = TinkerStorageGraph.open(conf);
+ graph.addVertex(T.id, 1, "value", 42);
+ graph.tx().commit();
+ graph.close();
+
+ graph = TinkerStorageGraph.open(conf);
+ assertEquals(1, countOf(graph.vertices()));
+ assertEquals(Integer.valueOf(42), graph.vertices(1).next().value("value"));
+ graph.close();
+ }
+
+ @Test
+ public void shouldPersistWithDefaultCommitSyncMode() {
+ // with no sync mode configured the engine defaults to 'commit' (fsync per commit); data must round-trip.
+ TinkerStorageGraph graph = open();
+ graph.addVertex(T.id, 1, "value", 42);
+ graph.tx().commit();
+ graph.close();
+
+ graph = open();
+ assertEquals(Integer.valueOf(42), graph.vertices(1).next().value("value"));
+ graph.close();
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldRejectUnknownSyncMode() {
+ final Configuration conf = config();
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, "bogus");
+ TinkerStorageGraph.open(conf);
+ }
+
+ @Test
+ public void shouldLeaveNoTempSnapshotAfterCompaction() {
+ final TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ graph.addVertex(T.id, 1, "value", 1);
+ graph.tx().commit();
+ graph.compact();
+ // the atomic rename must consume the temp file, leaving a durable snapshot and no leftover .tmp
+ assertTrue(new File(location, GraphBinaryStorage.SNAPSHOT_FILE).exists());
+ assertTrue(!new File(location, GraphBinaryStorage.SNAPSHOT_FILE + ".tmp").exists());
+ graph.close();
+ }
+
@Test
public void shouldRecoverFromTruncatedTrailingFrame() throws Exception {
TinkerStorageGraph graph = open();
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncModeTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncModeTest.java
new file mode 100644
index 0000000000..be1ddd2607
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncModeTest.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class SyncModeTest {
+
+ @Test
+ public void shouldDefaultToCommitWhenUnset() {
+ assertEquals(SyncMode.COMMIT, SyncMode.fromConfigValue(null));
+ }
+
+ @Test
+ public void shouldResolveCaseInsensitivelyAndTrim() {
+ assertEquals(SyncMode.COMMIT, SyncMode.fromConfigValue("commit"));
+ assertEquals(SyncMode.COMMIT, SyncMode.fromConfigValue("COMMIT"));
+ assertEquals(SyncMode.OS, SyncMode.fromConfigValue("os"));
+ assertEquals(SyncMode.OS, SyncMode.fromConfigValue(" Os "));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldRejectUnknownValue() {
+ SyncMode.fromConfigValue("interval:1000");
+ }
+}
(tinkerpop) 15/20: Document TinkerStorageGraph compact encoding and vertex-property id option
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 571488d52ef5a1966eda57d3e001b0faff27038b
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Wed Aug 19 14:41:17 2026 +0000
Document TinkerStorageGraph compact encoding and vertex-property id option
Add gremlin.tinkergraph.storage.preserveVertexPropertyIds to the TinkerGraph
configuration table and note in the persistence section that element and edge
ids are always preserved on reopen while auto-generated vertex-property ids are
regenerated by default. Fold the compact dictionary encoding and the new
option into the storage CHANGELOG entry.
Assisted-by: Claude Code:claude-opus-4-8
---
CHANGELOG.asciidoc | 2 +-
docs/src/reference/implementations-tinkergraph.asciidoc | 8 ++++++++
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 6a6b2bd684..fa611f6ddf 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,7 +28,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Fixed `gremlin-go` to report a malformed or truncated GraphBinary response as a deserialization error rather than a bare decoder message.
* Made `TinkerGraph` an interface and renamed the in-memory implementation to `TinkerMemoryGraph`; `TinkerGraph.open()` and `gremlin.graph=...TinkerGraph` behave as before. *(breaking)*
* Renamed `TinkerTransactionGraph` to `TinkerStorageGraph`. *(breaking)*
-* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast; the append log auto-compacts once it exceeds `gremlin.tinkergraph.storage.compactThresh [...]
+* Added a pluggable storage layer to `TinkerStorageGraph` that durably persists each committed transaction to disk, selected with the `gremlin.tinkergraph.storage` config key and shipping a GraphBinary engine, with a `gremlin.tinkergraph.storage.sync` key to choose `commit` (fsync per commit) or `os` durability; a storage location is locked to a single writer, so opening one already in use fails fast; the append log auto-compacts once it exceeds `gremlin.tinkergraph.storage.compactThresh [...]
* Removed automatic persistence from `TinkerMemoryGraph`, which is now purely in-memory and ignores `gremlin.tinkergraph.graphLocation`/`graphFormat`. Use `TinkerStorageGraph` for durability or `g.io()` for interchange. *(breaking)*
[[release-4-0-0-beta-3]]
diff --git a/docs/src/reference/implementations-tinkergraph.asciidoc b/docs/src/reference/implementations-tinkergraph.asciidoc
index b657d0c9f4..7f7db73fbe 100644
--- a/docs/src/reference/implementations-tinkergraph.asciidoc
+++ b/docs/src/reference/implementations-tinkergraph.asciidoc
@@ -186,6 +186,10 @@ may be lost on an operating system crash or power loss. Only meaningful when `gr
automatically compacts its append log, bounding the log growth and restart time of a long-running graph that is never
explicitly closed. Defaults to `67108864` (64 MB). A value of `0` disables automatic compaction, leaving it to
`close()` or an explicit `compact()`. Only meaningful when `gremlin.tinkergraph.storage` is set.
+|gremlin.tinkergraph.storage.preserveVertexPropertyIds |Whether a `TinkerStorageGraph` storage engine persists
+auto-generated vertex-property ids so they are stable across a close and reopen. Defaults to `false`, which
+regenerates those ids on load and keeps the store smaller. Element and edge ids are always preserved regardless. Only
+meaningful when `gremlin.tinkergraph.storage` is set.
|=========================================================
NOTE: To use <<tinkergraph-gremlin-tx, transactions>> and <<tinkergraph-gremlin-persistence, persistence>>, configure
@@ -456,6 +460,10 @@ committed state. Compaction runs when the graph is closed and can be requested e
unbounded log, compaction also runs automatically once the log grows past `gremlin.tinkergraph.storage.compactThreshold`
bytes. Setting that threshold to `0` disables automatic compaction.
+Element and edge ids are always preserved across a reopen. Auto-generated vertex-property ids are not, by default,
+so a vertex property may receive a different id after a reopen. Setting `gremlin.tinkergraph.storage.preserveVertexPropertyIds`
+to `true` persists those ids as well, at the cost of a larger store.
+
A storage location may be opened by only one graph at a time. `TinkerStorageGraph` takes an exclusive lock on the
storage directory when it opens, so a second attempt to open the same location, whether from the same JVM or another
process, fails rather than corrupting the data. The lock is released when the graph is closed. A store also records
(tinkerpop) 04/20: Serialize concurrent commit writes to TinkerStorageGraph storage
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 963217cf1b9db47baff177032f5216b067e511c2
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Mon Aug 3 20:53:41 2026 +0000
Serialize concurrent commit writes to TinkerStorageGraph storage
TinkerGraph transactions lock only their own changed elements, so commits
touching disjoint elements ran their commit paths concurrently and both wrote
to the storage engine's single append log at once, interleaving and corrupting
its records. A fair per-graph commit-write lock now serializes the engine's
persist/flush (and the compact/close paths that rewrite the same files), held
only around the durable write so disjoint commits still proceed in parallel up
to that point.
Assisted-by: Claude Code:claude-opus-4-8
---
.../tinkergraph/structure/AbstractTinkerGraph.java | 23 ++++-
.../tinkergraph/structure/TinkerStorageGraph.java | 11 ++-
.../tinkergraph/structure/TinkerTransaction.java | 12 ++-
.../tinkergraph/structure/storage/SyncMode.java | 9 +-
.../AbstractTinkerStorageConformanceTest.java | 53 +++++++++++
.../structure/storage/ConcurrencyProbeStorage.java | 82 ++++++++++++++++
.../storage/StorageCommitSerializationTest.java | 105 +++++++++++++++++++++
7 files changed, 285 insertions(+), 10 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
index ac1a3e6474..bb663ad0bd 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/AbstractTinkerGraph.java
@@ -45,6 +45,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantLock;
/**
* Base class for {@link TinkerMemoryGraph} and {@link TinkerStorageGraph}.
@@ -84,6 +85,15 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
*/
protected TinkerStorage storage;
+ /**
+ * Serializes the durable write of a committing transaction. TinkerGraph transactions lock only their own changed
+ * elements, so two commits touching disjoint elements run their commit paths concurrently; without this lock they
+ * would both write to the storage engine's single append log at once and interleave (corrupt) its records. Held
+ * only around the engine's persist/flush, so commits of disjoint elements still proceed in parallel up to that
+ * point. Fair, so committers are served in arrival order and none is starved.
+ */
+ protected final ReentrantLock storageCommitLock = new ReentrantLock(true);
+
/**
* Guard set while a graph is replaying its storage log on open. While {@code true}, mutations must not be
* re-persisted, otherwise replay would append the loaded data back to the log.
@@ -308,9 +318,16 @@ public abstract class AbstractTinkerGraph implements TinkerGraph {
@Override
public void close() {
if (storage != null) {
- storage.flush();
- storage.compact(this);
- storage.close();
+ // serialize against concurrent commit writes: close flushes, compacts, and closes the log, which must not
+ // interleave with a transaction appending to it.
+ storageCommitLock.lock();
+ try {
+ storage.flush();
+ storage.compact(this);
+ storage.close();
+ } finally {
+ storageCommitLock.unlock();
+ }
}
serviceRegistry.close();
GqlDeclarativeMatchStrategy.evict(this);
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
index 7dd54500a7..409f618f25 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerStorageGraph.java
@@ -303,8 +303,15 @@ public final class TinkerStorageGraph extends AbstractTinkerGraph {
*/
public void compact() {
if (storage != null) {
- storage.flush();
- storage.compact(this);
+ // hold the same lock as the commit write path: compaction closes the log, rewrites the snapshot, and
+ // truncates the log, which must not interleave with a concurrent transaction appending to that log.
+ storageCommitLock.lock();
+ try {
+ storage.flush();
+ storage.compact(this);
+ } finally {
+ storageCommitLock.unlock();
+ }
}
}
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
index 7bef8a9264..50c48bde80 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerTransaction.java
@@ -179,10 +179,16 @@ final class TinkerTransaction extends AbstractThreadLocalTransaction {
// write-ahead: durably persist the changeset before applying the in-memory commit, so a failure here
// aborts the commit (via the catch below) and leaves memory and disk consistent. Skipped while the graph
- // is replaying its storage log on open.
+ // is replaying its storage log on open. Serialized by storageCommitLock because commits of disjoint
+ // elements otherwise reach the engine's single append log concurrently and interleave its records.
if (graph.storage != null && !graph.loading) {
- graph.storage.persist(txVersion, toVertexMutations(changedVertices), toEdgeMutations(changedEdges));
- graph.storage.flush();
+ graph.storageCommitLock.lock();
+ try {
+ graph.storage.persist(txVersion, toVertexMutations(changedVertices), toEdgeMutations(changedEdges));
+ graph.storage.flush();
+ } finally {
+ graph.storageCommitLock.unlock();
+ }
}
// commit all changes
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
index de049797fe..d83e2d00a9 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/SyncMode.java
@@ -46,8 +46,13 @@ public enum SyncMode {
// TODO: add an INTERVAL mode (group commit) — a peer value on this same key, encoded as "interval:<ms>", that
// fsyncs at most once per <ms> window rather than once per commit, bounding the crash-loss window by time while
// amortizing fsync cost across commits. It implies fsync (a batched COMMIT), so it slots in as a third
- // mutually-exclusive mode without changing the meaning of COMMIT or OS. Deferred until concurrent commits are
- // serialized, since group commit's payoff is amortizing one fsync across many concurrent committers.
+ // mutually-exclusive mode without changing the meaning of COMMIT or OS.
+ //
+ // This is coupled to the commit-write lock that serializes concurrent commits (see the persist/flush step in
+ // TinkerTransaction.doCommit): once that lock exists, COMMIT holds it across the fsync, so every commit serializes
+ // on disk-sync latency. INTERVAL is the fix — hold the lock only for the buffer append (fast) and fsync one batch
+ // for many transactions. So INTERVAL should be built on top of that lock, not before it: it is the performance
+ // pass that makes serialized commits cheap, which is why it is deferred until concurrent commits are serialized.
/**
* Resolve a configuration value to a {@link SyncMode}, matched case-insensitively, defaulting to {@link #COMMIT}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
index 561320bcff..552cd37da4 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/AbstractTinkerStorageConformanceTest.java
@@ -32,7 +32,14 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -231,6 +238,52 @@ public abstract class AbstractTinkerStorageConformanceTest {
graph.close();
}
+ @Test
+ public void shouldPersistConcurrentCommitsWithoutLossAcrossReopen() throws Exception {
+ // End-to-end companion to StorageCommitSerializationTest: many threads commit disjoint vertices at once and,
+ // on reopen, every record must survive. This exercises the real engine but cannot by itself *prove* the lock
+ // works — log corruption from interleaving is scheduling-dependent — so the deterministic guarantee is
+ // asserted separately by StorageCommitSerializationTest via a probe engine.
+ final int threads = 8;
+ final int commitsPerThread = 50;
+ final TinkerStorageGraph writeGraph = open();
+ try {
+ final ExecutorService pool = Executors.newFixedThreadPool(threads);
+ final CountDownLatch start = new CountDownLatch(1);
+ final List<Future<?>> futures = new ArrayList<>();
+ for (int t = 0; t < threads; t++) {
+ final int threadId = t;
+ futures.add(pool.submit(() -> {
+ start.await(); // release all threads together to maximize contention on the commit path
+ for (int i = 0; i < commitsPerThread; i++) {
+ final int id = threadId * commitsPerThread + i;
+ writeGraph.addVertex(T.id, id, "value", id);
+ writeGraph.tx().commit();
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (final Future<?> f : futures)
+ f.get(60, TimeUnit.SECONDS);
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS));
+ } finally {
+ writeGraph.close();
+ }
+
+ // reopen from disk: a corrupt (interleaved) log frame would throw or drop records here
+ final TinkerStorageGraph reopened = open();
+ try {
+ final int expected = threads * commitsPerThread;
+ assertEquals(expected, countOf(reopened.vertices()));
+ for (int id = 0; id < expected; id++)
+ assertEquals(Integer.valueOf(id), reopened.vertices(id).next().value("value"));
+ } finally {
+ reopened.close();
+ }
+ }
+
private static long countOf(final Iterator<?> it) {
long count = 0;
while (it.hasNext()) {
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ConcurrencyProbeStorage.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ConcurrencyProbeStorage.java
new file mode 100644
index 0000000000..21b6c014da
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/ConcurrencyProbeStorage.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.AbstractTinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex;
+
+import java.util.Collection;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A {@link TinkerStorage} test double that deterministically detects whether the commit path serializes writes. It
+ * persists nothing; instead {@link #persist} tracks how many threads are inside it at once and records a violation if
+ * ever more than one is. It also sleeps briefly while "inside" to widen the window, so an unserialized commit path
+ * trips the detector reliably rather than racily. Selected by fully-qualified class name via
+ * {@code gremlin.tinkergraph.storage}. State is static because the engine is instantiated reflectively.
+ */
+public final class ConcurrencyProbeStorage implements TinkerStorage {
+
+ static final AtomicInteger inFlight = new AtomicInteger(0);
+ static final AtomicInteger maxObserved = new AtomicInteger(0);
+ static final AtomicInteger concurrentEntries = new AtomicInteger(0);
+
+ static void reset() {
+ inFlight.set(0);
+ maxObserved.set(0);
+ concurrentEntries.set(0);
+ }
+
+ @Override
+ public void open(final AbstractTinkerGraph graph, final Configuration config) { }
+
+ @Override
+ public void replay(final AbstractTinkerGraph graph) { }
+
+ @Override
+ public void persist(final long txVersion,
+ final Collection<TinkerStorageMutation<TinkerVertex>> changedVertices,
+ final Collection<TinkerStorageMutation<TinkerEdge>> changedEdges) {
+ final int concurrent = inFlight.incrementAndGet();
+ try {
+ maxObserved.accumulateAndGet(concurrent, Math::max);
+ if (concurrent > 1)
+ concurrentEntries.incrementAndGet();
+ // widen the window so an unserialized path is caught deterministically, not by luck
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException ie) {
+ Thread.currentThread().interrupt();
+ }
+ } finally {
+ inFlight.decrementAndGet();
+ }
+ }
+
+ @Override
+ public void flush() { }
+
+ @Override
+ public void compact(final AbstractTinkerGraph graph) { }
+
+ @Override
+ public void close() { }
+}
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCommitSerializationTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCommitSerializationTest.java
new file mode 100644
index 0000000000..2a0d5dd174
--- /dev/null
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/StorageCommitSerializationTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.tinkergraph.structure.storage;
+
+import org.apache.commons.configuration2.BaseConfiguration;
+import org.apache.commons.configuration2.Configuration;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerStorageGraph;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Deterministically verifies that {@link TinkerStorageGraph} serializes the durable write of concurrent commits.
+ * Because TinkerGraph transactions lock only their own changed elements, commits touching disjoint elements run their
+ * commit paths concurrently; the storage engine's single append log would interleave (corrupt) without the
+ * commit-write lock. Rather than rely on a race actually corrupting the log, this drives commits through
+ * {@link ConcurrencyProbeStorage}, which records whether two threads are ever inside {@code persist()} at once. With
+ * the lock that count is exactly zero; without it the probe trips reliably.
+ */
+public class StorageCommitSerializationTest {
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ private String location;
+
+ @Before
+ public void setUp() throws Exception {
+ location = tempFolder.newFolder("storage").getAbsolutePath();
+ ConcurrencyProbeStorage.reset();
+ }
+
+ private TinkerStorageGraph open() {
+ final Configuration conf = new BaseConfiguration();
+ conf.setProperty(Graph.GRAPH, TinkerStorageGraph.class.getName());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE, ConcurrencyProbeStorage.class.getName());
+ conf.setProperty(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, location);
+ return TinkerStorageGraph.open(conf);
+ }
+
+ @Test
+ public void shouldSerializeConcurrentCommitWrites() throws Exception {
+ final int threads = 8;
+ final int commitsPerThread = 20;
+ final TinkerStorageGraph graph = open();
+ try {
+ final ExecutorService pool = Executors.newFixedThreadPool(threads);
+ final CountDownLatch start = new CountDownLatch(1);
+ final List<Future<?>> futures = new ArrayList<>();
+ for (int t = 0; t < threads; t++) {
+ final int threadId = t;
+ futures.add(pool.submit(() -> {
+ start.await();
+ for (int i = 0; i < commitsPerThread; i++) {
+ graph.addVertex(T.id, threadId * commitsPerThread + i);
+ graph.tx().commit();
+ }
+ return null;
+ }));
+ }
+ start.countDown();
+ for (final Future<?> f : futures)
+ f.get(60, TimeUnit.SECONDS);
+ pool.shutdown();
+ assertTrue(pool.awaitTermination(60, TimeUnit.SECONDS));
+ } finally {
+ graph.close();
+ }
+
+ // the lock must have kept persist() strictly single-threaded
+ assertEquals("commits entered persist() concurrently", 0, ConcurrencyProbeStorage.concurrentEntries.get());
+ assertEquals("more than one thread was inside persist() at once", 1, ConcurrencyProbeStorage.maxObserved.get());
+ }
+}
(tinkerpop) 10/20: Record TinkerStorageGraph format version once per store
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch tinkergraph-storage
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 666a098a1173a1049acf4ee7dbde2aaff89276df
Author: Stephen Mallette <st...@amazon.com>
AuthorDate: Tue Aug 4 22:19:28 2026 +0000
Record TinkerStorageGraph format version once per store
The format version lived in every file header, so a store holding a snapshot
and log written at different times could carry two versions that disagree.
It now lives once in a store-level VERSION marker (magic + version, fsynced),
the single source of truth; per-file headers keep only the magic for identity
and corruption detection. Opening a store whose marker names an unsupported
version fails loudly and points to g.io() export as the migration path — the
engine deliberately does no in-place migration. Unknown record opcodes remain
a hard error, so a genuinely incompatible change bumps the version.
Assisted-by: Claude Code:claude-opus-4-8
---
.../structure/storage/GraphBinaryStorage.java | 89 +++++++++++++++++++---
.../structure/storage/GraphBinaryStorageTest.java | 24 ++++++
2 files changed, 101 insertions(+), 12 deletions(-)
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
index 715dc48c74..2aaa7643cc 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorage.java
@@ -67,6 +67,15 @@ import java.util.zip.CRC32;
* (default) {@code fsync}s so the commit survives an OS crash or power loss, while {@link SyncMode#OS} only flushes to
* the operating system.
* <p/>
+ * On-disk compatibility: the store's format version is recorded once in a {@code VERSION} marker file (magic +
+ * version), which is the single source of truth even when a store momentarily holds a snapshot and a log written at
+ * different times. Individual files carry only the magic for identity/corruption detection. Opening a store whose
+ * marker names an unsupported version fails loudly — records are never misread across a format change. The engine
+ * keeps this simple by not attempting in-place migration: the supported path across an incompatible format bump is to
+ * export via {@code g.io()} before upgrading. Additive changes should prefer new record opcodes; unknown opcodes are
+ * a hard error (a durable store must not silently drop records it cannot parse), so a genuinely incompatible change
+ * bumps the version.
+ * <p/>
* The in-memory graph remains authoritative (write-through). This engine does not support graphs larger than memory.
* <p/>
* Known limitation (write amplification): a commit records each changed element in full — a single property change on
@@ -84,15 +93,24 @@ public final class GraphBinaryStorage implements TinkerStorage {
static final byte[] MAGIC = { 'T', 'G', 'S', 'B' };
/**
- * On-disk format version, written once in each file's header after {@link #MAGIC}. A future format bump is
- * detected here so old files can be rejected (or, later, migrated) rather than misread.
+ * On-disk format version of the store. Recorded once per store in the {@link #VERSION_FILE} marker rather than in
+ * every file, so a store has a single unambiguous version even when it momentarily holds a snapshot and a log
+ * written at different times. A future format bump is detected against this marker so an older store is rejected
+ * (never silently misread); the supported migration path is to export via {@code g.io()} before upgrading.
*/
static final byte FORMAT_VERSION = 1;
/**
- * Bytes of the fixed file header: {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}.
+ * Store-level version marker file, holding {@link #MAGIC} followed by the one-byte {@link #FORMAT_VERSION}. It is
+ * the single source of truth for the store's format version.
+ */
+ static final String VERSION_FILE = "VERSION";
+
+ /**
+ * Bytes of the per-file header: just {@link #MAGIC}. The format version lives in the store-level
+ * {@link #VERSION_FILE}, not in each file.
*/
- static final int HEADER_SIZE = MAGIC.length + 1;
+ static final int HEADER_SIZE = MAGIC.length;
private static final byte OP_PUT_VERTEX = 1;
private static final byte OP_DEL_VERTEX = 2;
@@ -108,6 +126,7 @@ public final class GraphBinaryStorage implements TinkerStorage {
private File directory;
private File snapshotFile;
private File logFile;
+ private File versionFile;
/**
* Default automatic-compaction threshold: 64 MB of appended log since the last compaction.
@@ -130,12 +149,61 @@ public final class GraphBinaryStorage implements TinkerStorage {
this.directory = new File(location);
this.snapshotFile = new File(directory, SNAPSHOT_FILE);
this.logFile = new File(directory, LOG_FILE);
+ this.versionFile = new File(directory, VERSION_FILE);
this.syncMode = SyncMode.fromConfigValue(config.getString(TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_SYNC, null));
this.compactThresholdBytes = config.getLong(
TinkerGraph.GREMLIN_TINKERGRAPH_STORAGE_COMPACT_THRESHOLD, DEFAULT_COMPACT_THRESHOLD_BYTES);
// seed the counter with any pre-existing log so a graph reopened with a large log still compacts promptly
this.logBytesSinceCompaction = logFile.exists() ? logFile.length() : 0;
ensureDirectory();
+ establishStoreVersion();
+ }
+
+ /**
+ * Read and validate the store-level version marker, or create it for a new store. This is the single source of
+ * truth for the store's format version: a marker naming an unsupported version, or bad magic, fails the open
+ * loudly rather than risking a misread. The supported path across an incompatible format bump is to export the
+ * graph via {@code g.io()} before upgrading.
+ */
+ private void establishStoreVersion() {
+ // an existing store (has a snapshot or log) written before the marker existed is treated as version 1
+ final boolean storeHasData = snapshotFile.exists() || logFile.exists();
+ if (!versionFile.exists()) {
+ if (storeHasData && FORMAT_VERSION != 1)
+ throw new IllegalStateException(String.format(
+ "Storage location %s has data but no version marker; cannot confirm it is format version %d",
+ directory, FORMAT_VERSION));
+ writeStoreVersion();
+ return;
+ }
+ try (final DataInputStream in = new DataInputStream(new java.io.BufferedInputStream(new FileInputStream(versionFile)))) {
+ final byte[] magic = new byte[MAGIC.length];
+ readFully(in, magic);
+ if (!Arrays.equals(magic, MAGIC))
+ throw new IOException(String.format("%s is not a TinkerGraph storage version marker (bad magic)", versionFile));
+ final byte version = in.readByte();
+ if (version != FORMAT_VERSION)
+ throw new IOException(String.format(
+ "Unsupported storage format version %d at %s (this build writes %d); export via g.io() before upgrading",
+ version, directory, FORMAT_VERSION));
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not read storage version marker %s", versionFile), ex);
+ }
+ }
+
+ /**
+ * Write the store-level version marker ({@link #MAGIC} + {@link #FORMAT_VERSION}) durably.
+ */
+ private void writeStoreVersion() {
+ try (final FileOutputStream fos = new FileOutputStream(versionFile);
+ final DataOutputStream out = new DataOutputStream(fos)) {
+ out.write(MAGIC);
+ out.writeByte(FORMAT_VERSION);
+ out.flush();
+ fos.getFD().sync();
+ } catch (IOException ex) {
+ throw new UncheckedIOException(String.format("Could not write storage version marker %s", versionFile), ex);
+ }
}
/**
@@ -195,8 +263,9 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Read and validate the fixed file header, returning the number of record bytes that follow it. An empty file
- * (freshly created, no header yet) is treated as having no records.
+ * Read and validate the per-file header (magic only), returning the number of record bytes that follow it. An
+ * empty file (freshly created, no header yet) is treated as having no records. The store's format version is
+ * validated once against the {@link #VERSION_FILE} marker in {@link #open}, not here.
*/
private long readAndVerifyHeader(final DataInputStream in, final File file, final long fileLength) throws IOException {
if (fileLength == 0)
@@ -207,10 +276,6 @@ public final class GraphBinaryStorage implements TinkerStorage {
readFully(in, magic);
if (!Arrays.equals(magic, MAGIC))
throw new IOException(String.format("%s is not a TinkerGraph storage file (bad magic)", file));
- final byte version = in.readByte();
- if (version != FORMAT_VERSION)
- throw new IOException(String.format(
- "Unsupported storage format version %d in %s (expected %d)", version, file, FORMAT_VERSION));
return fileLength - HEADER_SIZE;
}
@@ -441,11 +506,11 @@ public final class GraphBinaryStorage implements TinkerStorage {
}
/**
- * Write the fixed file header ({@link #MAGIC} + {@link #FORMAT_VERSION}) at the start of a storage file.
+ * Write the per-file header ({@link #MAGIC}) at the start of a storage file. The format version is recorded once
+ * per store in the {@link #VERSION_FILE} marker, not per file.
*/
private static void writeHeader(final DataOutputStream out) throws IOException {
out.write(MAGIC);
- out.writeByte(FORMAT_VERSION);
}
private void closeLog() {
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
index 7daf2eb5ed..c113511176 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/storage/GraphBinaryStorageTest.java
@@ -360,6 +360,30 @@ public class GraphBinaryStorageTest extends AbstractTinkerStorageConformanceTest
}
}
+ @Test
+ public void shouldFailOnStoreWithUnsupportedVersionMarker() throws Exception {
+ TinkerStorageGraph graph = open();
+ final String location = graph.configuration().getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION);
+ graph.addVertex(T.id, 1);
+ graph.tx().commit();
+ graph.close();
+
+ // rewrite the store-level version marker to name a future, unsupported format version
+ final File versionFile = new File(location, GraphBinaryStorage.VERSION_FILE);
+ final byte[] marker = Files.readAllBytes(versionFile.toPath());
+ marker[marker.length - 1] = 99; // version byte follows the magic
+ Files.write(versionFile.toPath(), marker);
+
+ try {
+ open();
+ fail("expected reopen to fail on an unsupported store version");
+ } catch (Exception expected) {
+ assertTrue("cause should report the unsupported version and migration path: " + rootMessage(expected),
+ rootMessage(expected).contains("Unsupported storage format version")
+ && rootMessage(expected).contains("g.io()"));
+ }
+ }
+
private static String rootMessage(final Throwable t) {
Throwable cur = t;
while (cur.getCause() != null && cur.getCause() != cur)