You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2026/05/06 05:44:53 UTC
(fory) branch main updated: feat(xlang): add comprehensive read checks (#3647)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new 380fae71d feat(xlang): add comprehensive read checks (#3647)
380fae71d is described below
commit 380fae71dc732dad3e7936995157fbcab062cedf
Author: Shawn Yang <sh...@gmail.com>
AuthorDate: Wed May 6 13:44:48 2026 +0800
feat(xlang): add comprehensive read checks (#3647)
## Why?
Deserializers should reject malformed or attacker-controlled payloads
before they can drive invalid protocol state or oversized allocations.
This PR hardens root header, metadata, size, and string reads across the
xlang runtimes and updates the specs to match the validated wire format.
## What does this PR do?
- Standardizes the root header bitmap so xlang uses bit 0, out-of-band
uses bit 1, and reserved bits are rejected.
- Removes root-header null handling in favor of the normal root value
encoding path.
- Hardens TypeDef and TypeMeta parsing with 52-bit body hashes,
reserved/compression flag validation, root kind validation, and stricter
metadata cache/skip behavior.
- Adds read-side guardrails for binary payloads, primitive arrays,
collections/maps, strings, streams, varints, and trailing bytes across
Java, C++, C#, Dart, Go, JavaScript, Python, Rust, and Swift.
- Adds Java public configuration for max binary and collection
allocation sizes, and enables Rust UTF-8 string read validation by
default.
- Expands focused malformed-payload tests and updates Java/xlang
serialization specs to document the new header and metadata layouts.
## Related issues
#1017
## AI Contribution Checklist
- [ ] Substantial AI assistance was used in this PR: `yes` / `no`
- [ ] If `yes`, I included a completed [AI Contribution
Checklist](https://github.com/apache/fory/blob/main/AI_POLICY.md#9-contributor-checklist-for-ai-assisted-prs)
in this PR description and the required `AI Usage Disclosure`.
- [ ] If `yes`, my PR description includes the required `ai_review`
summary and screenshot evidence of the final clean AI review results
from both fresh reviewers on the current PR diff or current HEAD after
the latest code changes.
## Does this PR introduce any user-facing change?
- [x] Does this PR introduce any public API change?
- [x] Does this PR introduce any binary protocol compatibility change?
## Benchmark
Not applicable; this PR adds deserialization validation and protocol
tests rather than benchmark changes.
---
.agents/docs-and-formatting.md | 2 +
.agents/languages/java.md | 3 +
.gitignore | 1 +
AGENTS.md | 1 +
BUILD | 1 +
README.md | 6 +-
cpp/fory/serialization/BUILD | 1 +
cpp/fory/serialization/context.cc | 40 +-
cpp/fory/serialization/fory.h | 51 +-
cpp/fory/serialization/serialization_test.cc | 226 ++-
cpp/fory/serialization/serializer.h | 12 +-
cpp/fory/serialization/type_resolver.cc | 382 +++--
cpp/fory/serialization/type_resolver.h | 3 +-
csharp/src/Fory/Fory.cs | 62 +-
csharp/src/Fory/ForyFlags.cs | 6 +-
csharp/src/Fory/MetaString.cs | 2 +
csharp/src/Fory/ReadContext.cs | 54 +-
csharp/src/Fory/StringSerializer.cs | 3 +-
csharp/src/Fory/TypeInfo.cs | 9 +-
csharp/src/Fory/TypeMeta.cs | 244 ++-
csharp/src/Fory/TypeResolver.cs | 16 +
csharp/tests/Fory.Tests/ForyRuntimeTests.cs | 65 +-
csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs | 91 +-
.../fory/lib/src/context/read_context.dart | 33 +-
dart/packages/fory/lib/src/fory.dart | 41 +-
dart/packages/fory/lib/src/meta/meta_string.dart | 2 +
dart/packages/fory/lib/src/meta/type_meta.dart | 42 +-
.../fory/lib/src/resolver/type_resolver.dart | 226 ++-
dart/packages/fory/lib/src/util/hash_util.dart | 9 +-
.../fory/test/decimal_serializer_test.dart | 4 +-
dart/packages/fory/test/time_serializer_test.dart | 2 +-
dart/packages/fory/test/xlang_protocol_test.dart | 78 +-
docs/specification/java_serialization_spec.md | 35 +-
docs/specification/xlang_serialization_spec.md | 46 +-
go/fory/buffer.go | 37 +-
go/fory/buffer_test.go | 27 +
go/fory/decimal.go | 4 +-
go/fory/enum.go | 22 +-
go/fory/enum_test.go | 98 ++
go/fory/fory.go | 180 +-
go/fory/fory_typed_test.go | 38 +
go/fory/map.go | 8 +-
go/fory/map_primitive.go | 261 +--
go/fory/map_primitive_test.go | 61 +
go/fory/meta_string_resolver.go | 122 +-
go/fory/meta_string_resolver_test.go | 125 ++
go/fory/primitive.go | 60 +-
go/fory/reader.go | 143 +-
go/fory/skip.go | 137 +-
go/fory/skip_test.go | 132 ++
go/fory/stream.go | 39 +-
go/fory/stream_test.go | 23 +
go/fory/string.go | 12 +-
go/fory/struct.go | 4 +-
go/fory/struct_test.go | 17 +-
go/fory/tests/metastring_resolver_test.go | 2 +-
go/fory/time.go | 12 +-
go/fory/type_def.go | 388 +++--
go/fory/type_def_test.go | 218 ++-
go/fory/type_resolver.go | 18 +-
go/fory/writer.go | 4 +-
.../idl_tests/javascript/roundtrip.ts | 6 +-
.../src/main/java/org/apache/fory/Fory.java | 47 +-
.../apache/fory/collection/LongLongByteMap.java | 20 +
.../main/java/org/apache/fory/config/Config.java | 18 +
.../java/org/apache/fory/config/ForyBuilder.java | 26 +
.../org/apache/fory/context/MetaStringReader.java | 118 +-
.../java/org/apache/fory/context/ReadContext.java | 10 +-
.../org/apache/fory/io/ForyReadableChannel.java | 28 +-
.../java/org/apache/fory/memory/MemoryBuffer.java | 102 +-
.../apache/fory/meta/DeflaterMetaCompressor.java | 9 +
.../org/apache/fory/meta/EncodedMetaString.java | 6 +-
.../java/org/apache/fory/meta/MetaCompressor.java | 10 +
.../org/apache/fory/meta/NativeTypeDefDecoder.java | 155 +-
.../org/apache/fory/meta/NativeTypeDefEncoder.java | 78 +-
.../main/java/org/apache/fory/meta/TypeDef.java | 53 +-
.../java/org/apache/fory/meta/TypeDefDecoder.java | 145 +-
.../java/org/apache/fory/meta/TypeDefEncoder.java | 95 +-
.../apache/fory/meta/TypeEqualMetaCompressor.java | 5 +
.../org/apache/fory/resolver/ClassResolver.java | 114 +-
.../org/apache/fory/resolver/SharedRegistry.java | 45 +-
.../org/apache/fory/resolver/TypeNameBytes.java | 18 +-
.../org/apache/fory/resolver/TypeResolver.java | 86 +-
.../org/apache/fory/resolver/XtypeResolver.java | 17 +-
.../apache/fory/serializer/ArraySerializers.java | 35 +
.../fory/serializer/BigIntegerSerializer.java | 17 +-
.../apache/fory/serializer/BufferSerializers.java | 7 +
.../apache/fory/serializer/DecimalSerializer.java | 36 +-
.../fory/serializer/ExceptionSerializers.java | 5 +
.../fory/serializer/ObjectStreamSerializer.java | 66 +-
.../fory/serializer/PrimitiveArraySerializers.java | 183 +-
.../collection/ChildContainerSerializers.java | 10 +-
.../collection/CollectionLikeSerializer.java | 29 +-
.../collection/CollectionSerializers.java | 93 +-
.../collection/GuavaCollectionSerializers.java | 14 +-
.../collection/ImmutableCollectionSerializers.java | 6 +-
.../serializer/collection/MapLikeSerializer.java | 28 +-
.../fory/serializer/collection/MapSerializers.java | 16 +-
.../collection/PrimitiveListSerializers.java | 93 +-
.../serializer/collection/SubListSerializers.java | 2 +-
.../src/test/java/org/apache/fory/ForyTest.java | 8 +
.../src/test/java/org/apache/fory/StreamTest.java | 61 +
.../org/apache/fory/memory/MemoryBufferTest.java | 19 +
.../fory/meta/DeflaterMetaCompressorTest.java | 11 +
.../apache/fory/meta/NativeTypeDefEncoderTest.java | 175 +-
.../org/apache/fory/meta/TypeDefEncoderTest.java | 197 +++
.../java/org/apache/fory/meta/TypeDefTest.java | 2 +-
.../apache/fory/resolver/ClassResolverTest.java | 41 +-
.../org/apache/fory/resolver/MetaStringIOTest.java | 89 +
.../fory/serializer/ArraySerializersTest.java | 119 ++
.../fory/serializer/BufferSerializersTest.java | 43 +
.../fory/serializer/JdkProxySerializerTest.java | 26 +
.../serializer/ObjectStreamSerializerTest.java | 55 +-
.../fory/serializer/PrimitiveSerializersTest.java | 61 +
.../apache/fory/serializer/SerializersTest.java | 51 +
.../collection/CollectionSerializersTest.java | 69 +
.../serializer/collection/MapSerializersTest.java | 20 +
.../fory/extension/meta/TypeDefEncoderTest.java | 3 +-
javascript/packages/core/lib/context.ts | 102 +-
javascript/packages/core/lib/fory.ts | 86 +-
javascript/packages/core/lib/meta/TypeMeta.ts | 468 +++--
javascript/packages/core/lib/reader/index.ts | 123 +-
javascript/packages/core/lib/type.ts | 5 +-
javascript/test/datetime.test.ts | 41 +-
javascript/test/decimal.test.ts | 51 +-
javascript/test/fory.test.ts | 10 +-
javascript/test/hps.test.ts | 2 +-
javascript/test/io.test.ts | 23 +-
javascript/test/typemeta.test.ts | 219 ++-
python/pyfory/_fory.py | 30 +-
python/pyfory/context.pxi | 89 +-
python/pyfory/context.py | 18 +-
python/pyfory/format/infer.py | 10 +-
python/pyfory/format/tests/test_infer.py | 42 +
python/pyfory/meta/typedef.py | 175 +-
python/pyfory/meta/typedef_decoder.py | 84 +-
python/pyfory/meta/typedef_encoder.py | 64 +-
python/pyfory/registry.py | 16 +-
python/pyfory/serialization.pyx | 59 +-
python/pyfory/serializer.py | 81 +-
python/pyfory/tests/test_metastring_resolver.py | 113 +-
python/pyfory/tests/test_policy.py | 227 ++-
python/pyfory/tests/test_ref_tracking.py | 6 +-
python/pyfory/tests/test_root_header.py | 59 +
python/pyfory/tests/test_serializer.py | 2 +-
python/pyfory/tests/test_size_guardrails.py | 2 +-
python/pyfory/tests/test_typedef_encoding.py | 98 +-
python/pyfory/type_util.py | 10 +-
rust/fory-core/src/buffer.rs | 14 +-
rust/fory-core/src/config.rs | 9 +
rust/fory-core/src/context.rs | 8 +
rust/fory-core/src/fory.rs | 95 +-
rust/fory-core/src/meta/meta_string.rs | 20 +-
rust/fory-core/src/meta/type_meta.rs | 326 +++-
rust/fory-core/src/resolver/meta_resolver.rs | 67 +-
rust/fory-core/src/resolver/type_resolver.rs | 24 +-
rust/fory-core/src/serializer/string.rs | 9 +-
rust/fory-core/src/type_id.rs | 5 +-
rust/fory-core/src/util/string_util.rs | 20 +-
rust/tests/tests/test_buffer.rs | 36 +
rust/tests/tests/test_cross_language.rs | 2 +-
rust/tests/tests/test_meta.rs | 2 +-
swift/Sources/Fory/Fory.swift | 1026 ++++++-----
swift/Sources/Fory/ForyFlags.swift | 30 +-
swift/Sources/Fory/ReadContext.swift | 6 +
swift/Sources/Fory/TypeMeta.swift | 1275 +++++++-------
swift/Sources/Fory/TypeResolver.swift | 1231 +++++++-------
swift/Tests/ForyTests/DecimalTests.swift | 150 +-
swift/Tests/ForyTests/ForySwiftTests.swift | 1787 ++++++++++----------
169 files changed, 10560 insertions(+), 4789 deletions(-)
diff --git a/.agents/docs-and-formatting.md b/.agents/docs-and-formatting.md
index f4aaed7d1..0f9bfe258 100644
--- a/.agents/docs-and-formatting.md
+++ b/.agents/docs-and-formatting.md
@@ -31,6 +31,8 @@ Load this file when changing documentation, public APIs, protocol specs, benchma
## Formatting Commands
- Markdown: `prettier --write <file>`
+- Do not format Markdown under `tasks/`, including task design, plan, progress, state, history,
+ and lessons files. These files are agent working state rather than repository documentation.
- Python code, including `compiler/`, `benchmarks/`, `integration_tests/`, and `python/`:
`python -m ruff format <changed-python-files>` and
`python -m ruff check --fix <changed-python-files>`
diff --git a/.agents/languages/java.md b/.agents/languages/java.md
index 1d1957442..5283d5a4c 100644
--- a/.agents/languages/java.md
+++ b/.agents/languages/java.md
@@ -22,6 +22,9 @@ Load this file when changing anything under `java/` or when Java drives a cross-
- Do not add normal-JVM process-global caches keyed by user classes, generated classes, serializer classes, classloaders, or class-bound method handles. Prefer per-runtime state, immutable shared metadata, or build-time-only template data.
- Concrete serializers may opt into sharing only after auditing retained fields. Treat serializers retaining `TypeResolver`, `RefResolver`, mutable scratch buffers, runtime state, or classloader-sensitive state as non-shareable unless that state is externalized.
- Resolver and serializer hot paths should keep the fast-path/null-slow-path shape obvious. Hoist repeated buffer or cache-state access into locals for multi-step operations and keep rebuild/restoration logic cold.
+- In Java codec hot paths, avoid `Preconditions.checkArgument` for attacker-controlled primitive
+ validation. Use direct primitive branches and throw on the cold error path to preserve inlining and
+ avoid varargs/helper overhead.
- Keep GraalVM feature code as a thin metadata/registration layer. Build time should publish metadata needed for runtime reconstruction, not retain concrete generated or user serializer instances in the image heap.
- If changes touch GraalVM bootstrap, serializer retention, native-image metadata, or `ObjectStreamSerializer` GraalVM behavior, verify the native-image build and run the produced binary; a plain Java compile is insufficient.
- Put latest-JDK or virtual-thread tests in the latest-JDK test modules with the matching compiler/profile floor, and centralize runtime-version probing in existing compatibility utilities.
diff --git a/.gitignore b/.gitignore
index c979f24d9..daaab45bd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -133,6 +133,7 @@ test.md
benchmarks/dart/profile_output
**/*.fory.dart
+integration_tests/idl_tests/dart/.dart_tool/
**/pubspec.lock
**/tmp/*
\ No newline at end of file
diff --git a/AGENTS.md b/AGENTS.md
index bb92c1387..492560d23 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -104,6 +104,7 @@ This is the entry point for AI guidance in Apache Fory. Read this file first, th
## Shared Validation Expectations
- Run the relevant tests for every touched language or subsystem before finishing.
+- When multiple independent language test suites are required, run them concurrently when the environment has enough resources instead of running them one by one; keep each language's logs and results separate, and rerun any failed suite with focused diagnostics.
- Run applicable test commands in a subagent with a thinking budget one level lower than the main task budget, using medium when the current budget is unclear, unless the change is docs-only or the user explicitly asks to run them locally.
- Reuse the same test subagent for repeated runs within one task and subsystem so it keeps failure context; create a fresh subagent when switching unrelated subsystems or when prior context may be stale or misleading.
- Use `integration_tests/` for cross-language compatibility validation when behavior crosses runtimes.
diff --git a/BUILD b/BUILD
index 69350bd55..9928ab77c 100644
--- a/BUILD
+++ b/BUILD
@@ -52,6 +52,7 @@ pyx_library(
"//cpp/fory/type:fory_type",
"//python/pyfory/cpp:_pyfory",
"//cpp/fory/thirdparty:flat_hash_map",
+ "//cpp/fory/thirdparty:libmmh3",
],
)
diff --git a/README.md b/README.md
index 2890af49b..ba7a38769 100644
--- a/README.md
+++ b/README.md
@@ -118,7 +118,7 @@ For more detailed benchmarks and methodology, see [Go Benchmark](benchmarks/go).
<img src="docs/benchmarks/python/throughput.png" width="95%">
</p>
-For more detailed benchmarks and methodology, see [Pythonk](benchmarks/python).
+For more detailed benchmarks and methodology, see [Python](benchmarks/python).
### JavaScript/NodeJS Serialization Performance
@@ -744,7 +744,7 @@ Apache Fory™ supports class schema forward/backward compatibility across **Jav
### Binary Compatibility
-**Current Status**: Binary compatibility is **not guaranteed** between Fory major releases as the protocol continues to evolve. However, compatibility **is guaranteed** between minor versions (e.g., 0.13.x).
+**Current Status**: Binary compatibility is **not guaranteed** between Fory major releases as the protocol continues to evolve. Compatibility **is guaranteed** between minor versions (for example, 0.13.x).
**Recommendations**:
@@ -752,7 +752,7 @@ Apache Fory™ supports class schema forward/backward compatibility across **Jav
- Plan migration strategies when upgrading major versions
- See [upgrade guide](docs/guide/java) for details
-**Future**: Binary compatibility will be guaranteed starting from Fory 1.0 release.
+Major-version compatibility is the boundary for stable serialized data.
## Security
diff --git a/cpp/fory/serialization/BUILD b/cpp/fory/serialization/BUILD
index 87b3ede02..5da749d9e 100644
--- a/cpp/fory/serialization/BUILD
+++ b/cpp/fory/serialization/BUILD
@@ -51,6 +51,7 @@ cc_test(
srcs = ["serialization_test.cc"],
deps = [
":fory_serialization",
+ "//cpp/fory/thirdparty:libmmh3",
"@googletest//:gtest",
"@googletest//:gtest_main",
],
diff --git a/cpp/fory/serialization/context.cc b/cpp/fory/serialization/context.cc
index 7967a3335..946d37d8f 100644
--- a/cpp/fory/serialization/context.cc
+++ b/cpp/fory/serialization/context.cc
@@ -500,48 +500,28 @@ Result<const TypeInfo *, Error> ReadContext::read_type_meta() {
// Check if we already parsed this type meta (cache lookup by header)
if (has_last_meta_header_ && meta_header == last_meta_header_) {
- // Fast path: same header as last parsed
+ // Header-cache hits intentionally skip without rehashing. Entries reach
+ // this cache only after a successful TypeMeta parse and 52-bit body-hash
+ // validation.
const TypeInfo *cached = last_meta_type_info_;
reading_type_infos_.push_back(cached);
- if (cached && !cached->type_def.empty()) {
- const size_t type_def_size = cached->type_def.size();
- if (type_def_size >= sizeof(int64_t) &&
- type_def_size <= std::numeric_limits<uint32_t>::max()) {
- Error skip_error;
- buffer_->skip(static_cast<uint32_t>(type_def_size - sizeof(int64_t)),
- skip_error);
- if (FORY_PREDICT_FALSE(!skip_error.ok())) {
- return Unexpected(std::move(skip_error));
- }
- return cached;
- }
- }
- FORY_RETURN_NOT_OK(TypeMeta::skip_bytes(*buffer_, meta_header));
+ FORY_RETURN_NOT_OK(
+ TypeMeta::skip_bytes_for_validated_header(*buffer_, meta_header));
return cached;
}
auto *cache_entry = parsed_type_infos_.find(meta_header);
if (cache_entry != nullptr) {
- // Found in cache - reuse and skip the bytes
+ // Header-cache hits intentionally skip without rehashing. Entries reach
+ // this cache only after a successful TypeMeta parse and 52-bit body-hash
+ // validation.
const TypeInfo *cached = cache_entry->second;
reading_type_infos_.push_back(cached);
has_last_meta_header_ = true;
last_meta_header_ = meta_header;
last_meta_type_info_ = cached;
- if (cached && !cached->type_def.empty()) {
- const size_t type_def_size = cached->type_def.size();
- if (type_def_size >= sizeof(int64_t) &&
- type_def_size <= std::numeric_limits<uint32_t>::max()) {
- Error skip_error;
- buffer_->skip(static_cast<uint32_t>(type_def_size - sizeof(int64_t)),
- skip_error);
- if (FORY_PREDICT_FALSE(!skip_error.ok())) {
- return Unexpected(std::move(skip_error));
- }
- return cached;
- }
- }
- FORY_RETURN_NOT_OK(TypeMeta::skip_bytes(*buffer_, meta_header));
+ FORY_RETURN_NOT_OK(
+ TypeMeta::skip_bytes_for_validated_header(*buffer_, meta_header));
return cached;
}
diff --git a/cpp/fory/serialization/fory.h b/cpp/fory/serialization/fory.h
index fee0c71aa..8edc1c50a 100644
--- a/cpp/fory/serialization/fory.h
+++ b/cpp/fory/serialization/fory.h
@@ -629,15 +629,13 @@ public:
Buffer buffer(const_cast<uint8_t *>(data), static_cast<uint32_t>(size),
false);
- FORY_TRY(header, read_header(buffer));
- if (header.is_null) {
- return Unexpected(Error::invalid_data("Cannot deserialize null object"));
+ Error header_error;
+ const uint8_t header = buffer.read_uint8(header_error);
+ if (FORY_PREDICT_FALSE(!header_error.ok())) {
+ return Unexpected(std::move(header_error));
}
- if (FORY_PREDICT_FALSE(header.is_xlang != config_.xlang)) {
- return Unexpected(Error::invalid_data(
- "Protocol mismatch: payload xlang=" +
- std::string(header.is_xlang ? "true" : "false") +
- ", local xlang=" + std::string(config_.xlang ? "true" : "false")));
+ if (FORY_PREDICT_FALSE(header != precomputed_header_)) {
+ return Unexpected(invalid_root_header(header));
}
read_ctx_->attach(buffer);
@@ -668,19 +666,13 @@ public:
if (FORY_PREDICT_FALSE(!finalized_)) {
ensure_finalized();
}
- auto header_result = read_header(buffer);
- if (FORY_PREDICT_FALSE(!header_result.ok())) {
- return Unexpected(std::move(header_result).error());
+ Error header_error;
+ const uint8_t header = buffer.read_uint8(header_error);
+ if (FORY_PREDICT_FALSE(!header_error.ok())) {
+ return Unexpected(std::move(header_error));
}
- auto header = std::move(header_result).value();
- if (header.is_null) {
- return Unexpected(Error::invalid_data("Cannot deserialize null object"));
- }
- if (FORY_PREDICT_FALSE(header.is_xlang != config_.xlang)) {
- return Unexpected(Error::invalid_data(
- "Protocol mismatch: payload xlang=" +
- std::string(header.is_xlang ? "true" : "false") +
- ", local xlang=" + std::string(config_.xlang ? "true" : "false")));
+ if (FORY_PREDICT_FALSE(header != precomputed_header_)) {
+ return Unexpected(invalid_root_header(header));
}
read_ctx_->attach(buffer);
@@ -775,11 +767,28 @@ private:
static uint8_t compute_header(bool xlang) {
uint8_t flags = 0;
if (xlang) {
- flags |= (1 << 1); // bit 1: xlang flag
+ flags |= (1 << 0);
}
return flags;
}
+ FORY_NOINLINE Error invalid_root_header(uint8_t header) const {
+ constexpr uint8_t xlang_flag = 1 << 0;
+ constexpr uint8_t oob_flag = 1 << 1;
+ constexpr uint8_t known_flags = xlang_flag | oob_flag;
+ if ((header & ~known_flags) != 0) {
+ return Error::invalid_data("Unsupported root header bitmap");
+ }
+ if ((header & oob_flag) != 0) {
+ return Error::invalid_data("Out-of-band mode is not supported");
+ }
+ const bool payload_xlang = (header & xlang_flag) != 0;
+ return Error::invalid_data(
+ "Protocol mismatch: payload xlang=" +
+ std::string(payload_xlang ? "true" : "false") +
+ ", local xlang=" + std::string(config_.xlang ? "true" : "false"));
+ }
+
/// Core serialization implementation.
/// TypeMeta is written inline using streaming protocol (no deferred writing).
template <typename T>
diff --git a/cpp/fory/serialization/serialization_test.cc b/cpp/fory/serialization/serialization_test.cc
index c5c51e3f7..4fb31c5dd 100644
--- a/cpp/fory/serialization/serialization_test.cc
+++ b/cpp/fory/serialization/serialization_test.cc
@@ -20,6 +20,7 @@
#include "fory/serialization/fory.h"
#include "fory/serialization/ref_resolver.h"
#include "fory/serialization/skip.h"
+#include "fory/thirdparty/MurmurHash3.h"
#include "gtest/gtest.h"
#include <atomic>
#include <chrono>
@@ -68,8 +69,8 @@ struct NestedStruct {
};
enum class Color { RED, GREEN, BLUE };
-enum class LegacyStatus : int32_t { NEG = -3, ZERO = 0, LARGE = 42 };
-FORY_ENUM(LegacyStatus, NEG, ZERO, LARGE);
+enum class SignedScopedStatus : int32_t { NEG = -3, ZERO = 0, LARGE = 42 };
+FORY_ENUM(SignedScopedStatus, NEG, ZERO, LARGE);
enum class SparseStatus : int32_t { UNKNOWN = 4096, OK = 8192 };
FORY_ENUM(SparseStatus, UNKNOWN, OK);
@@ -80,6 +81,23 @@ namespace fory {
namespace serialization {
namespace test {
+namespace {
+
+uint64_t compute_type_meta_hash_bits_for_test(const uint8_t *meta_bytes,
+ size_t meta_size) {
+ constexpr uint32_t kHashShift = 12;
+ constexpr uint64_t kHashBitsMask = UINT64_MAX << kHashShift;
+ int64_t hash_out[2] = {0, 0};
+ MurmurHash3_x64_128(meta_bytes, static_cast<int>(meta_size), 47, hash_out);
+ uint64_t shifted = static_cast<uint64_t>(hash_out[0]) << kHashShift;
+ if (static_cast<int64_t>(shifted) < 0) {
+ shifted = ~shifted + 1;
+ }
+ return shifted & kHashBitsMask;
+}
+
+} // namespace
+
// ============================================================================
// Test Helpers
// ============================================================================
@@ -95,7 +113,7 @@ inline void register_test_types(Fory &fory) {
// Register all enum types used in tests
fory.register_enum<Color>(type_id++);
- fory.register_enum<LegacyStatus>(type_id++);
+ fory.register_enum<SignedScopedStatus>(type_id++);
fory.register_enum<SparseStatus>(type_id++);
fory.register_enum<OldStatus>(type_id++);
}
@@ -224,7 +242,7 @@ TEST(SerializationTest, DateExposesDaysSinceEpochAccessorAndRoundTrips) {
std::vector<uint8_t> bytes = std::move(serialize_result).value();
Buffer expected;
- expected.write_uint8(0b10);
+ expected.write_uint8(0b1);
expected.write_int8(NOT_NULL_VALUE_FLAG);
expected.write_uint8(static_cast<uint8_t>(TypeId::DATE));
expected.write_var_int64(-1);
@@ -308,7 +326,7 @@ TEST(SerializationTest, DecimalRejectsNonCanonicalBigPayloads) {
auto fory = Fory::builder().xlang(true).track_ref(false).build();
Buffer zero_big_encoding;
- zero_big_encoding.write_uint8(0b10);
+ zero_big_encoding.write_uint8(0b1);
zero_big_encoding.write_int8(NOT_NULL_VALUE_FLAG);
zero_big_encoding.write_uint8(static_cast<uint8_t>(TypeId::DECIMAL));
zero_big_encoding.write_var_int32(0);
@@ -322,7 +340,7 @@ TEST(SerializationTest, DecimalRejectsNonCanonicalBigPayloads) {
std::string::npos);
Buffer trailing_zero_payload;
- trailing_zero_payload.write_uint8(0b10);
+ trailing_zero_payload.write_uint8(0b1);
trailing_zero_payload.write_int8(NOT_NULL_VALUE_FLAG);
trailing_zero_payload.write_uint8(static_cast<uint8_t>(TypeId::DECIMAL));
trailing_zero_payload.write_var_int32(0);
@@ -459,9 +477,9 @@ TEST(SerializationTest, SparseEnumRoundtrip) {
TEST(SerializationTest, EnumSerializesOrdinalValue) {
auto fory = Fory::builder().xlang(true).track_ref(false).build();
- fory.register_enum<LegacyStatus>(1);
+ fory.register_enum<SignedScopedStatus>(1);
- auto bytes_result = fory.serialize(LegacyStatus::LARGE);
+ auto bytes_result = fory.serialize(SignedScopedStatus::LARGE);
ASSERT_TRUE(bytes_result.ok())
<< "Serialization failed: " << bytes_result.error().to_string();
@@ -500,9 +518,9 @@ TEST(SerializationTest, OldEnumSerializesOrdinalValue) {
TEST(SerializationTest, EnumOrdinalMappingHandlesNonZeroStart) {
auto fory = Fory::builder().xlang(true).track_ref(false).build();
- fory.register_enum<LegacyStatus>(1);
+ fory.register_enum<SignedScopedStatus>(1);
- auto bytes_result = fory.serialize(LegacyStatus::NEG);
+ auto bytes_result = fory.serialize(SignedScopedStatus::NEG);
ASSERT_TRUE(bytes_result.ok())
<< "Serialization failed: " << bytes_result.error().to_string();
@@ -516,17 +534,18 @@ TEST(SerializationTest, EnumOrdinalMappingHandlesNonZeroStart) {
// Ordinal 0 encoded as varuint32 is just 1 byte with value 0
EXPECT_EQ(bytes[offset + 3], 0);
- auto roundtrip = fory.deserialize<LegacyStatus>(bytes.data(), bytes.size());
+ auto roundtrip =
+ fory.deserialize<SignedScopedStatus>(bytes.data(), bytes.size());
ASSERT_TRUE(roundtrip.ok())
<< "Deserialization failed: " << roundtrip.error().to_string();
- EXPECT_EQ(roundtrip.value(), LegacyStatus::NEG);
+ EXPECT_EQ(roundtrip.value(), SignedScopedStatus::NEG);
}
TEST(SerializationTest, EnumOrdinalMappingRejectsInvalidOrdinal) {
auto fory = Fory::builder().xlang(true).track_ref(false).build();
- fory.register_enum<LegacyStatus>(1);
+ fory.register_enum<SignedScopedStatus>(1);
- auto bytes_result = fory.serialize(LegacyStatus::NEG);
+ auto bytes_result = fory.serialize(SignedScopedStatus::NEG);
ASSERT_TRUE(bytes_result.ok())
<< "Serialization failed: " << bytes_result.error().to_string();
@@ -536,7 +555,8 @@ TEST(SerializationTest, EnumOrdinalMappingRejectsInvalidOrdinal) {
// offset + 3 Replace the valid ordinal with an invalid one (99 as varuint32)
bytes[offset + 3] = 99;
- auto decode = fory.deserialize<LegacyStatus>(bytes.data(), bytes.size());
+ auto decode =
+ fory.deserialize<SignedScopedStatus>(bytes.data(), bytes.size());
EXPECT_FALSE(decode.ok());
}
@@ -689,6 +709,41 @@ TEST(SerializationTest, DeserializeRejectsXlangProtocolMismatch) {
std::string::npos);
}
+TEST(SerializationTest, RootHeaderUsesXlangBitZero) {
+ auto fory = Fory::builder().xlang(true).build();
+ auto bytes_result = fory.serialize<int32_t>(123);
+ ASSERT_TRUE(bytes_result.ok())
+ << "Serialization failed: " << bytes_result.error().to_string();
+ ASSERT_FALSE(bytes_result.value().empty());
+ EXPECT_EQ(bytes_result.value()[0], 0x01);
+}
+
+TEST(SerializationTest, DeserializeRejectsRootHeaderReservedBits) {
+ auto fory = Fory::builder().xlang(true).build();
+ auto bytes_result = fory.serialize<int32_t>(123);
+ ASSERT_TRUE(bytes_result.ok())
+ << "Serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ bytes[0] = 0x05;
+ auto result = fory.deserialize<int32_t>(bytes.data(), bytes.size());
+ ASSERT_FALSE(result.ok());
+ EXPECT_EQ(result.error().code(), ErrorCode::InvalidData);
+}
+
+TEST(SerializationTest, DeserializeRejectsOutOfBandRootHeader) {
+ auto fory = Fory::builder().xlang(true).build();
+ auto bytes_result = fory.serialize<int32_t>(123);
+ ASSERT_TRUE(bytes_result.ok())
+ << "Serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ bytes[0] = 0x03;
+ auto result = fory.deserialize<int32_t>(bytes.data(), bytes.size());
+ ASSERT_FALSE(result.ok());
+ EXPECT_EQ(result.error().code(), ErrorCode::InvalidData);
+}
+
TEST(SerializationTest, RegistrationByIdFailureDoesNotLeakTypeInfo) {
auto fory = Fory::builder().xlang(true).track_ref(false).build();
TypeResolver &resolver = fory.type_resolver();
@@ -756,6 +811,147 @@ TEST(SerializationTest, TypeMetaRejectsOverConsumedDeclaredSize) {
EXPECT_EQ(parsed.error().code(), ErrorCode::InvalidData);
}
+TEST(SerializationTest, TypeMetaHeaderUses52BitBodyHash) {
+ std::vector<FieldInfo> fields;
+ fields.emplace_back(
+ "value", FieldType(static_cast<uint32_t>(TypeId::VARINT32), false));
+ TypeMeta meta = TypeMeta::from_fields(static_cast<uint32_t>(TypeId::STRUCT),
+ "", "S", false, 1, std::move(fields));
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ const std::vector<uint8_t> &bytes = bytes_result.value();
+ ASSERT_GT(bytes.size(), sizeof(uint64_t));
+ uint64_t header = 0;
+ std::memcpy(&header, bytes.data(), sizeof(header));
+
+ constexpr uint64_t kMetaSizeMask = 0xff;
+ constexpr uint64_t kCompressMetaFlag = 0x100;
+ constexpr uint64_t kReservedBitsMask = 0xe00;
+ constexpr uint32_t kHashShift = 12;
+
+ EXPECT_EQ(header & kCompressMetaFlag, 0);
+ EXPECT_EQ(header & kReservedBitsMask, 0);
+ ASSERT_NE(header & kMetaSizeMask, kMetaSizeMask);
+ uint64_t meta_size = header & kMetaSizeMask;
+ ASSERT_EQ(bytes.size(), sizeof(uint64_t) + meta_size);
+ ASSERT_GT(meta_size, 0);
+ uint8_t body_header = bytes[sizeof(uint64_t)];
+ EXPECT_EQ(body_header & 0x80, 0x80);
+ EXPECT_EQ(body_header & 0x40, 0);
+ EXPECT_EQ(body_header & 0x20, 0);
+ EXPECT_EQ(body_header & 0x1F, 1);
+
+ std::vector<uint8_t> parse_bytes = bytes;
+ Buffer buffer(parse_bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_TRUE(parsed.ok()) << parsed.error().to_string();
+ EXPECT_EQ(static_cast<int64_t>(header >> kHashShift),
+ parsed.value()->get_hash());
+}
+
+TEST(SerializationTest, TypeMetaNonStructHeaderUsesDenseKindCode) {
+ TypeMeta meta =
+ TypeMeta::from_fields(static_cast<uint32_t>(TypeId::ENUM), "", "E", false,
+ 7, std::vector<FieldInfo>{});
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ ASSERT_GT(bytes.size(), sizeof(uint64_t));
+ EXPECT_EQ(bytes[sizeof(uint64_t)], 0x00);
+
+ Buffer buffer(bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_TRUE(parsed.ok()) << parsed.error().to_string();
+ EXPECT_EQ(parsed.value()->get_type_id(), static_cast<uint32_t>(TypeId::ENUM));
+}
+
+TEST(SerializationTest, TypeMetaRejectsNonStructReservedKindBits) {
+ TypeMeta meta =
+ TypeMeta::from_fields(static_cast<uint32_t>(TypeId::ENUM), "", "E", false,
+ 7, std::vector<FieldInfo>{});
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ bytes[sizeof(uint64_t)] |= 0x10;
+ uint64_t header = 0;
+ std::memcpy(&header, bytes.data(), sizeof(header));
+ ASSERT_NE(header & 0xff, 0xff);
+ header &= ~(UINT64_MAX << 12);
+ header |= compute_type_meta_hash_bits_for_test(
+ bytes.data() + sizeof(uint64_t), bytes.size() - sizeof(uint64_t));
+ std::memcpy(bytes.data(), &header, sizeof(header));
+
+ Buffer buffer(bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_FALSE(parsed.ok());
+ EXPECT_EQ(parsed.error().code(), ErrorCode::InvalidData);
+ EXPECT_NE(parsed.error().to_string().find("kind header"), std::string::npos);
+}
+
+TEST(SerializationTest, TypeMetaRejectsReservedHeaderBits) {
+ TypeMeta meta =
+ TypeMeta::from_fields(static_cast<uint32_t>(TypeId::STRUCT), "", "S",
+ false, 1, std::vector<FieldInfo>{});
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ uint64_t header = 0;
+ std::memcpy(&header, bytes.data(), sizeof(header));
+ header |= 0x200;
+ std::memcpy(bytes.data(), &header, sizeof(header));
+
+ Buffer buffer(bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_FALSE(parsed.ok());
+ EXPECT_EQ(parsed.error().code(), ErrorCode::InvalidData);
+}
+
+TEST(SerializationTest, TypeMetaRejectsUnsupportedCompressedHeader) {
+ TypeMeta meta =
+ TypeMeta::from_fields(static_cast<uint32_t>(TypeId::STRUCT), "", "S",
+ false, 1, std::vector<FieldInfo>{});
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ uint64_t header = 0;
+ std::memcpy(&header, bytes.data(), sizeof(header));
+ header |= 0x100;
+ std::memcpy(bytes.data(), &header, sizeof(header));
+
+ Buffer buffer(bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_FALSE(parsed.ok());
+ EXPECT_EQ(parsed.error().code(), ErrorCode::InvalidData);
+}
+
+TEST(SerializationTest, TypeMetaRejectsBodyHashMismatchAfterParse) {
+ TypeMeta meta =
+ TypeMeta::from_fields(static_cast<uint32_t>(TypeId::STRUCT), "", "S",
+ false, 1, std::vector<FieldInfo>{});
+ auto bytes_result = meta.to_bytes();
+ ASSERT_TRUE(bytes_result.ok())
+ << "TypeMeta serialization failed: " << bytes_result.error().to_string();
+
+ std::vector<uint8_t> bytes = bytes_result.value();
+ ASSERT_GT(bytes.size(), sizeof(uint64_t));
+ bytes.back() ^= 0x01;
+
+ Buffer buffer(bytes);
+ auto parsed = TypeMeta::from_bytes(buffer, nullptr);
+ ASSERT_FALSE(parsed.ok());
+ EXPECT_EQ(parsed.error().code(), ErrorCode::InvalidData);
+}
+
// ============================================================================
// Configuration Tests
// ============================================================================
diff --git a/cpp/fory/serialization/serializer.h b/cpp/fory/serialization/serializer.h
index 36477101a..8b6a7b15a 100644
--- a/cpp/fory/serialization/serializer.h
+++ b/cpp/fory/serialization/serializer.h
@@ -72,7 +72,6 @@ inline bool is_little_endian_system() {
/// Fory header information
struct HeaderInfo {
- bool is_null;
bool is_xlang;
bool is_oob;
uint32_t meta_start_offset; // 0 if not present
@@ -89,9 +88,14 @@ inline Result<HeaderInfo, Error> read_header(Buffer &buffer) {
return Unexpected(std::move(error));
}
HeaderInfo info;
- info.is_null = (flags & (1 << 0)) != 0;
- info.is_xlang = (flags & (1 << 1)) != 0;
- info.is_oob = (flags & (1 << 2)) != 0;
+ constexpr uint8_t xlang_flag = 1 << 0;
+ constexpr uint8_t oob_flag = 1 << 1;
+ constexpr uint8_t known_flags = xlang_flag | oob_flag;
+ if (FORY_PREDICT_FALSE((flags & ~known_flags) != 0)) {
+ return Unexpected(Error::invalid_data("Unsupported root header bitmap"));
+ }
+ info.is_xlang = (flags & xlang_flag) != 0;
+ info.is_oob = (flags & oob_flag) != 0;
// Note: Meta start offset would be read here if present
info.meta_start_offset = 0;
diff --git a/cpp/fory/serialization/type_resolver.cc b/cpp/fory/serialization/type_resolver.cc
index 86074a7f7..b5ffb5917 100644
--- a/cpp/fory/serialization/type_resolver.cc
+++ b/cpp/fory/serialization/type_resolver.cc
@@ -25,6 +25,7 @@
#include <algorithm>
#include <cstring>
#include <functional>
+#include <limits>
#include <map>
#include <unordered_map>
@@ -36,13 +37,18 @@ using namespace meta;
// Constants from xlang spec
constexpr size_t SMALL_NUM_FIELDS_THRESHOLD = 0b11111;
constexpr uint8_t REGISTER_BY_NAME_FLAG = 0b100000;
+constexpr uint8_t COMPATIBLE_TYPEDEF_FLAG = 0b01000000;
+constexpr uint8_t STRUCT_TYPEDEF_FLAG = 0b10000000;
+constexpr uint8_t NON_STRUCT_RESERVED_BITS_MASK = 0b01110000;
constexpr size_t FIELD_NAME_SIZE_THRESHOLD = 0b1111;
constexpr size_t BIG_NAME_THRESHOLD = 0b111111;
-constexpr int64_t META_SIZE_MASK = 0xff;
-// Temporary xlang behavior: keep TypeMeta uncompressed because some runtimes
-// still do not support TypeMeta decompression.
-constexpr int64_t HAS_FIELDS_META_FLAG = 0b1 << 8;
-constexpr int8_t NUM_HASH_BITS = 50;
+constexpr uint64_t META_SIZE_MASK = 0xff;
+constexpr uint64_t COMPRESS_META_FLAG = 0x100;
+constexpr uint64_t TYPE_META_RESERVED_BITS_MASK = 0xe00;
+constexpr int8_t NUM_HASH_BITS = 52;
+constexpr uint32_t TYPE_META_HASH_SHIFT = 64 - NUM_HASH_BITS;
+constexpr uint64_t TYPE_META_HASH_BITS_MASK = ~uint64_t{0}
+ << TYPE_META_HASH_SHIFT;
// ============================================================================
// FieldType Implementation
@@ -189,7 +195,6 @@ Result<std::vector<uint8_t>, Error> FieldInfo::to_bytes() const {
buffer.write_bytes(encoded_name.data(), encoded_name.size());
}
- // CRITICAL FIX: Use writer_index() not size() to get actual bytes written!
return std::vector<uint8_t>(buffer.data(),
buffer.data() + buffer.writer_index());
}
@@ -329,6 +334,122 @@ write_meta_name(Buffer &buffer, const std::string &name,
return Result<void, Error>();
}
+inline bool is_compatible_struct_type_id(uint32_t type_id) {
+ return type_id == static_cast<uint32_t>(TypeId::COMPATIBLE_STRUCT) ||
+ type_id == static_cast<uint32_t>(TypeId::NAMED_COMPATIBLE_STRUCT);
+}
+
+inline Result<uint8_t, Error> type_meta_kind_code(uint32_t type_id) {
+ switch (static_cast<TypeId>(type_id)) {
+ case TypeId::ENUM:
+ return static_cast<uint8_t>(0);
+ case TypeId::NAMED_ENUM:
+ return static_cast<uint8_t>(1);
+ case TypeId::EXT:
+ return static_cast<uint8_t>(2);
+ case TypeId::NAMED_EXT:
+ return static_cast<uint8_t>(3);
+ case TypeId::TYPED_UNION:
+ return static_cast<uint8_t>(4);
+ case TypeId::NAMED_UNION:
+ return static_cast<uint8_t>(5);
+ default:
+ return Unexpected(Error::type_error("Unsupported TypeMeta kind"));
+ }
+}
+
+inline Result<uint32_t, Error> type_id_from_type_meta_kind(uint8_t kind_code) {
+ switch (kind_code) {
+ case 0:
+ return static_cast<uint32_t>(TypeId::ENUM);
+ case 1:
+ return static_cast<uint32_t>(TypeId::NAMED_ENUM);
+ case 2:
+ return static_cast<uint32_t>(TypeId::EXT);
+ case 3:
+ return static_cast<uint32_t>(TypeId::NAMED_EXT);
+ case 4:
+ return static_cast<uint32_t>(TypeId::TYPED_UNION);
+ case 5:
+ return static_cast<uint32_t>(TypeId::NAMED_UNION);
+ default:
+ return Unexpected(Error::invalid_data("Unsupported TypeMeta kind code"));
+ }
+}
+
+inline uint64_t compute_type_meta_hash_bits(const uint8_t *meta_bytes,
+ size_t meta_size) {
+ int64_t hash_out[2] = {0, 0};
+ MurmurHash3_x64_128(meta_bytes, static_cast<int>(meta_size), 47, hash_out);
+ uint64_t shifted = static_cast<uint64_t>(hash_out[0]) << TYPE_META_HASH_SHIFT;
+ if (static_cast<int64_t>(shifted) < 0) {
+ shifted = ~shifted + 1;
+ }
+ return shifted & TYPE_META_HASH_BITS_MASK;
+}
+
+inline int64_t compute_type_meta_hash(const uint8_t *meta_bytes,
+ size_t meta_size) {
+ return static_cast<int64_t>(
+ compute_type_meta_hash_bits(meta_bytes, meta_size) >>
+ TYPE_META_HASH_SHIFT);
+}
+
+inline Result<void, Error> validate_type_meta_header(uint64_t header) {
+ if (FORY_PREDICT_FALSE((header & TYPE_META_RESERVED_BITS_MASK) != 0)) {
+ return Unexpected(
+ Error::invalid_data("TypeMeta reserved header bits must be zero"));
+ }
+ if (FORY_PREDICT_FALSE((header & COMPRESS_META_FLAG) != 0)) {
+ return Unexpected(
+ Error::invalid_data("Compressed TypeMeta is not supported"));
+ }
+ return Result<void, Error>();
+}
+
+inline Result<uint32_t, Error>
+read_type_meta_size(Buffer &buffer, uint64_t header, size_t *header_size) {
+ Error error;
+ uint64_t meta_size = header & META_SIZE_MASK;
+ if (meta_size == META_SIZE_MASK) {
+ uint32_t before = buffer.reader_index();
+ uint32_t extra = buffer.read_var_uint32(error);
+ if (FORY_PREDICT_FALSE(!error.ok())) {
+ return Unexpected(std::move(error));
+ }
+ meta_size += extra;
+ uint32_t after = buffer.reader_index();
+ if (header_size != nullptr) {
+ *header_size += (after - before);
+ }
+ }
+ if (FORY_PREDICT_FALSE(
+ meta_size > static_cast<uint64_t>(std::numeric_limits<int>::max()))) {
+ return Unexpected(
+ Error::invalid_data("TypeMeta body size exceeds supported range"));
+ }
+ return static_cast<uint32_t>(meta_size);
+}
+
+inline Result<void, Error> validate_type_meta_hash(Buffer &buffer,
+ uint32_t body_start,
+ uint32_t meta_size,
+ int64_t header_hash) {
+ uint64_t body_end = static_cast<uint64_t>(body_start) + meta_size;
+ if (FORY_PREDICT_FALSE(body_end > buffer.reader_index() ||
+ body_end > buffer.size())) {
+ return Unexpected(
+ Error::invalid_data("TypeMeta body range is not readable"));
+ }
+ uint64_t computed_hash_bits = compute_type_meta_hash_bits(
+ buffer.data() + body_start, static_cast<size_t>(meta_size));
+ if (FORY_PREDICT_FALSE((computed_hash_bits >> TYPE_META_HASH_SHIFT) !=
+ static_cast<uint64_t>(header_hash))) {
+ return Unexpected(Error::invalid_data("TypeMeta body hash mismatch"));
+ }
+ return Result<void, Error>();
+}
+
inline Result<std::string, Error>
read_meta_name(Buffer &buffer, const MetaStringDecoder &decoder,
const MetaEncoding *encodings, size_t enc_count) {
@@ -392,21 +513,33 @@ TypeMeta TypeMeta::from_fields(uint32_t tid, const std::string &ns,
Result<std::vector<uint8_t>, Error> TypeMeta::to_bytes() const {
Buffer layer_buffer;
- // write meta header
+ bool is_struct = is_struct_type(static_cast<TypeId>(type_id));
size_t num_fields = field_infos.size();
- uint8_t meta_header =
- static_cast<uint8_t>(std::min(num_fields, SMALL_NUM_FIELDS_THRESHOLD));
- if (register_by_name) {
- meta_header |= REGISTER_BY_NAME_FLAG;
+ if (FORY_PREDICT_FALSE(!is_struct && num_fields != 0)) {
+ return Unexpected(
+ Error::invalid_data("Non-struct TypeMeta cannot carry field metadata"));
}
- layer_buffer.write_uint8(meta_header);
- if (num_fields >= SMALL_NUM_FIELDS_THRESHOLD) {
- layer_buffer.write_var_uint32(num_fields - SMALL_NUM_FIELDS_THRESHOLD);
+ if (is_struct) {
+ uint8_t meta_header =
+ STRUCT_TYPEDEF_FLAG |
+ static_cast<uint8_t>(std::min(num_fields, SMALL_NUM_FIELDS_THRESHOLD));
+ if (is_compatible_struct_type_id(type_id)) {
+ meta_header |= COMPATIBLE_TYPEDEF_FLAG;
+ }
+ if (register_by_name) {
+ meta_header |= REGISTER_BY_NAME_FLAG;
+ }
+ layer_buffer.write_uint8(meta_header);
+
+ if (num_fields >= SMALL_NUM_FIELDS_THRESHOLD) {
+ layer_buffer.write_var_uint32(num_fields - SMALL_NUM_FIELDS_THRESHOLD);
+ }
+ } else {
+ FORY_TRY(kind_code, type_meta_kind_code(type_id));
+ layer_buffer.write_uint8(kind_code);
}
- // write namespace and type name (if registered by name) using the
- // same compact meta string format as Rust/Java.
if (register_by_name) {
FORY_RETURN_NOT_OK(write_meta_name(
layer_buffer, namespace_str, k_namespace_encoder,
@@ -417,7 +550,6 @@ Result<std::vector<uint8_t>, Error> TypeMeta::to_bytes() const {
k_type_name_encodings,
sizeof(k_type_name_encodings) / sizeof(k_type_name_encodings[0])));
} else {
- layer_buffer.write_uint8(static_cast<uint8_t>(type_id));
if (user_type_id == kInvalidUserTypeId) {
return Unexpected(
Error::type_error("User type id is required for this type"));
@@ -434,26 +566,23 @@ Result<std::vector<uint8_t>, Error> TypeMeta::to_bytes() const {
// Now write global binary header
Buffer result_buffer;
const uint32_t layer_size = layer_buffer.writer_index();
- int64_t meta_size = layer_size;
- int64_t header = std::min(META_SIZE_MASK, meta_size);
-
- bool write_meta_fields_flag = !field_infos.empty();
- if (write_meta_fields_flag) {
- header |= HAS_FIELDS_META_FLAG;
+ if (FORY_PREDICT_FALSE(layer_size > static_cast<uint32_t>(
+ std::numeric_limits<int>::max()))) {
+ return Unexpected(
+ Error::invalid_data("TypeMeta body size exceeds supported range"));
}
+ uint64_t meta_size = layer_size;
+ uint64_t header = std::min(META_SIZE_MASK, meta_size);
- // Compute hash
- std::vector<uint8_t> layer_data(layer_buffer.data(),
- layer_buffer.data() + layer_size);
- int64_t meta_hash = compute_hash(layer_data);
- header |= (meta_hash << (64 - NUM_HASH_BITS));
+ header |= compute_type_meta_hash_bits(layer_buffer.data(), layer_size);
result_buffer.write_bytes(reinterpret_cast<const uint8_t *>(&header),
sizeof(header));
if (meta_size >= META_SIZE_MASK) {
- result_buffer.write_var_uint32(meta_size - META_SIZE_MASK);
+ result_buffer.write_var_uint32(
+ static_cast<uint32_t>(meta_size - META_SIZE_MASK));
}
- result_buffer.write_bytes(layer_data.data(), layer_data.size());
+ result_buffer.write_bytes(layer_buffer.data(), layer_size);
// Use actual bytes written to construct return vector
return std::vector<uint8_t>(result_buffer.data(),
result_buffer.data() +
@@ -473,39 +602,52 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) {
}
size_t header_size = sizeof(header);
- int64_t meta_size = header & META_SIZE_MASK;
- if (meta_size == META_SIZE_MASK) {
- uint32_t before = buffer.reader_index();
- uint32_t extra = buffer.read_var_uint32(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- meta_size += extra;
- uint32_t after = buffer.reader_index();
- header_size += (after - before);
- }
- int64_t meta_hash = header >> (64 - NUM_HASH_BITS);
+ uint64_t header_bits = static_cast<uint64_t>(header);
+ FORY_RETURN_IF_ERROR(validate_type_meta_header(header_bits));
+ FORY_TRY(meta_size, read_type_meta_size(buffer, header_bits, &header_size));
+ int64_t meta_hash = static_cast<int64_t>(header_bits >> TYPE_META_HASH_SHIFT);
+ uint32_t body_start = static_cast<uint32_t>(start_pos + header_size);
// Read meta header
uint8_t meta_header = buffer.read_uint8(error);
if (FORY_PREDICT_FALSE(!error.ok())) {
return Unexpected(std::move(error));
}
- bool register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0;
- size_t num_fields = meta_header & SMALL_NUM_FIELDS_THRESHOLD;
- if (num_fields == SMALL_NUM_FIELDS_THRESHOLD) {
- uint32_t extra = buffer.read_var_uint32(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- num_fields += extra;
- }
-
- // Read type ID or namespace/type name
uint32_t type_id = 0;
uint32_t user_type_id = kInvalidUserTypeId;
std::string namespace_str;
std::string type_name;
+ bool register_by_name = false;
+ size_t num_fields = 0;
+
+ if ((meta_header & STRUCT_TYPEDEF_FLAG) != 0) {
+ register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0;
+ bool compatible = (meta_header & COMPATIBLE_TYPEDEF_FLAG) != 0;
+ if (register_by_name) {
+ type_id = static_cast<uint32_t>(
+ compatible ? TypeId::NAMED_COMPATIBLE_STRUCT : TypeId::NAMED_STRUCT);
+ } else {
+ type_id = static_cast<uint32_t>(compatible ? TypeId::COMPATIBLE_STRUCT
+ : TypeId::STRUCT);
+ }
+ num_fields = meta_header & SMALL_NUM_FIELDS_THRESHOLD;
+ if (num_fields == SMALL_NUM_FIELDS_THRESHOLD) {
+ uint32_t extra = buffer.read_var_uint32(error);
+ if (FORY_PREDICT_FALSE(!error.ok())) {
+ return Unexpected(std::move(error));
+ }
+ num_fields += extra;
+ }
+ } else {
+ if (FORY_PREDICT_FALSE((meta_header & NON_STRUCT_RESERVED_BITS_MASK) !=
+ 0)) {
+ return Unexpected(Error::invalid_data("Invalid TypeMeta kind header"));
+ }
+ FORY_TRY(decoded_type_id,
+ type_id_from_type_meta_kind(meta_header & 0b1111));
+ type_id = decoded_type_id;
+ register_by_name = is_namespaced_type(static_cast<TypeId>(type_id));
+ }
if (register_by_name) {
static const MetaStringDecoder k_namespace_decoder('.', '_');
@@ -523,11 +665,6 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) {
sizeof(k_type_name_encodings[0])));
type_name = std::move(tn);
} else {
- uint32_t tid = buffer.read_uint8(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- type_id = tid;
uint32_t uid = buffer.read_var_uint32(error);
if (FORY_PREDICT_FALSE(!error.ok())) {
return Unexpected(std::move(error));
@@ -552,20 +689,18 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) {
assign_field_ids(local_type_info, field_infos);
}
- // CRITICAL FIX: Ensure we consume exactly meta_size bytes
size_t current_pos = buffer.reader_index();
size_t expected_end_pos = start_pos + header_size + meta_size;
if (FORY_PREDICT_FALSE(current_pos > expected_end_pos)) {
return Unexpected(Error::invalid_data(
"TypeMeta parser consumed beyond declared meta size"));
}
- if (current_pos < expected_end_pos) {
- size_t remaining = expected_end_pos - current_pos;
- buffer.skip(static_cast<uint32_t>(remaining), error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
+ if (FORY_PREDICT_FALSE(current_pos < expected_end_pos)) {
+ return Unexpected(Error::invalid_data(
+ "TypeMeta parser did not consume declared meta size"));
}
+ FORY_RETURN_IF_ERROR(
+ validate_type_meta_hash(buffer, body_start, meta_size, meta_hash));
auto meta = std::make_unique<TypeMeta>();
meta->hash = meta_hash;
@@ -581,18 +716,13 @@ TypeMeta::from_bytes(Buffer &buffer, const TypeMeta *local_type_info) {
Result<std::unique_ptr<TypeMeta>, Error>
TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) {
- Error error;
- int64_t meta_size = header & META_SIZE_MASK;
- if (meta_size == META_SIZE_MASK) {
- uint32_t extra = buffer.read_var_uint32(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- meta_size += extra;
- }
- int64_t meta_hash = header >> (64 - NUM_HASH_BITS);
+ uint64_t header_bits = static_cast<uint64_t>(header);
+ FORY_RETURN_IF_ERROR(validate_type_meta_header(header_bits));
+ FORY_TRY(meta_size, read_type_meta_size(buffer, header_bits, nullptr));
+ int64_t meta_hash = static_cast<int64_t>(header_bits >> TYPE_META_HASH_SHIFT);
- size_t start_pos = buffer.reader_index();
+ uint32_t start_pos = buffer.reader_index();
+ Error error;
// Read meta header
uint8_t meta_header = buffer.read_uint8(error);
@@ -600,21 +730,41 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) {
return Unexpected(std::move(error));
}
- bool register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0;
- size_t num_fields = meta_header & SMALL_NUM_FIELDS_THRESHOLD;
- if (num_fields == SMALL_NUM_FIELDS_THRESHOLD) {
- uint32_t extra = buffer.read_var_uint32(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- num_fields += extra;
- }
-
- // Read type ID or namespace/type name
uint32_t type_id = 0;
uint32_t user_type_id = kInvalidUserTypeId;
std::string namespace_str;
std::string type_name;
+ bool register_by_name = false;
+ size_t num_fields = 0;
+
+ if ((meta_header & STRUCT_TYPEDEF_FLAG) != 0) {
+ register_by_name = (meta_header & REGISTER_BY_NAME_FLAG) != 0;
+ bool compatible = (meta_header & COMPATIBLE_TYPEDEF_FLAG) != 0;
+ if (register_by_name) {
+ type_id = static_cast<uint32_t>(
+ compatible ? TypeId::NAMED_COMPATIBLE_STRUCT : TypeId::NAMED_STRUCT);
+ } else {
+ type_id = static_cast<uint32_t>(compatible ? TypeId::COMPATIBLE_STRUCT
+ : TypeId::STRUCT);
+ }
+ num_fields = meta_header & SMALL_NUM_FIELDS_THRESHOLD;
+ if (num_fields == SMALL_NUM_FIELDS_THRESHOLD) {
+ uint32_t extra = buffer.read_var_uint32(error);
+ if (FORY_PREDICT_FALSE(!error.ok())) {
+ return Unexpected(std::move(error));
+ }
+ num_fields += extra;
+ }
+ } else {
+ if (FORY_PREDICT_FALSE((meta_header & NON_STRUCT_RESERVED_BITS_MASK) !=
+ 0)) {
+ return Unexpected(Error::invalid_data("Invalid TypeMeta kind header"));
+ }
+ FORY_TRY(decoded_type_id,
+ type_id_from_type_meta_kind(meta_header & 0b1111));
+ type_id = decoded_type_id;
+ register_by_name = is_namespaced_type(static_cast<TypeId>(type_id));
+ }
if (register_by_name) {
static const MetaStringDecoder k_namespace_decoder('.', '_');
@@ -632,27 +782,11 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) {
sizeof(k_type_name_encodings[0])));
type_name = std::move(tn);
} else {
- uint32_t tid = buffer.read_uint8(error);
+ uint32_t uid = buffer.read_var_uint32(error);
if (FORY_PREDICT_FALSE(!error.ok())) {
return Unexpected(std::move(error));
}
- type_id = tid;
- switch (static_cast<TypeId>(type_id)) {
- case TypeId::ENUM:
- case TypeId::STRUCT:
- case TypeId::COMPATIBLE_STRUCT:
- case TypeId::EXT:
- case TypeId::TYPED_UNION: {
- uint32_t uid = buffer.read_var_uint32(error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
- user_type_id = uid;
- break;
- }
- default:
- break;
- }
+ user_type_id = uid;
}
// Read field infos
@@ -666,20 +800,18 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) {
// NOTE: Do NOT sort remote fields! They are already in the sender's sorted
// order, which matches the data order.
- // CRITICAL FIX: Ensure we consume exactly meta_size bytes
size_t current_pos = buffer.reader_index();
- size_t expected_end_pos = start_pos + meta_size;
+ size_t expected_end_pos = static_cast<size_t>(start_pos) + meta_size;
if (FORY_PREDICT_FALSE(current_pos > expected_end_pos)) {
return Unexpected(Error::invalid_data(
"TypeMeta parser consumed beyond declared meta size"));
}
- if (current_pos < expected_end_pos) {
- size_t remaining = expected_end_pos - current_pos;
- buffer.skip(static_cast<uint32_t>(remaining), error);
- if (FORY_PREDICT_FALSE(!error.ok())) {
- return Unexpected(std::move(error));
- }
+ if (FORY_PREDICT_FALSE(current_pos < expected_end_pos)) {
+ return Unexpected(Error::invalid_data(
+ "TypeMeta parser did not consume declared meta size"));
}
+ FORY_RETURN_IF_ERROR(
+ validate_type_meta_hash(buffer, start_pos, meta_size, meta_hash));
auto meta = std::make_unique<TypeMeta>();
meta->hash = meta_hash;
@@ -693,9 +825,10 @@ TypeMeta::from_bytes_with_header(Buffer &buffer, int64_t header) {
return meta;
}
-Result<void, Error> TypeMeta::skip_bytes(Buffer &buffer, int64_t header) {
+Result<void, Error> TypeMeta::skip_bytes_for_validated_header(Buffer &buffer,
+ int64_t header) {
Error error;
- int64_t meta_size = header & META_SIZE_MASK;
+ uint64_t meta_size = static_cast<uint64_t>(header) & META_SIZE_MASK;
if (meta_size == META_SIZE_MASK) {
uint32_t extra = buffer.read_var_uint32(error);
if (FORY_PREDICT_FALSE(!error.ok())) {
@@ -703,6 +836,12 @@ Result<void, Error> TypeMeta::skip_bytes(Buffer &buffer, int64_t header) {
}
meta_size += extra;
}
+ if (FORY_PREDICT_FALSE(
+ meta_size >
+ static_cast<uint64_t>(std::numeric_limits<uint32_t>::max()))) {
+ return Unexpected(
+ Error::invalid_data("TypeMeta body size exceeds supported range"));
+ }
buffer.skip(static_cast<uint32_t>(meta_size), error);
if (FORY_PREDICT_FALSE(!error.ok())) {
return Unexpected(std::move(error));
@@ -1026,18 +1165,7 @@ void TypeMeta::assign_field_ids(const TypeMeta *local_type,
}
int64_t TypeMeta::compute_hash(const std::vector<uint8_t> &meta_bytes) {
- // Compute hash using MurmurHash3_x64_128 to match Rust/Java
- // TypeMeta implementation. We take the high 64 bits and then
- // keep only the lower NUM_HASH_BITS bits.
- int64_t hash_out[2] = {0, 0};
- MurmurHash3_x64_128(meta_bytes.data(), static_cast<int>(meta_bytes.size()),
- 47, hash_out);
-
- // hash_out[0] is the low 64 bits, hash_out[1] the high 64 bits.
- uint64_t high = static_cast<uint64_t>(hash_out[1]);
- uint64_t mask = (NUM_HASH_BITS >= 64) ? ~uint64_t{0}
- : ((uint64_t{1} << NUM_HASH_BITS) - 1);
- return static_cast<int64_t>(high & mask);
+ return compute_type_meta_hash(meta_bytes.data(), meta_bytes.size());
}
namespace {
diff --git a/cpp/fory/serialization/type_resolver.h b/cpp/fory/serialization/type_resolver.h
index 85d928d56..cf4212632 100644
--- a/cpp/fory/serialization/type_resolver.h
+++ b/cpp/fory/serialization/type_resolver.h
@@ -319,7 +319,8 @@ public:
from_bytes_with_header(Buffer &buffer, int64_t header);
/// skip type meta in buffer without parsing
- static Result<void, Error> skip_bytes(Buffer &buffer, int64_t header);
+ static Result<void, Error> skip_bytes_for_validated_header(Buffer &buffer,
+ int64_t header);
/// Check struct version consistency
static Result<void, Error> check_struct_version(int32_t read_version,
diff --git a/csharp/src/Fory/Fory.cs b/csharp/src/Fory/Fory.cs
index 60bf46c09..032fec11e 100644
--- a/csharp/src/Fory/Fory.cs
+++ b/csharp/src/Fory/Fory.cs
@@ -16,6 +16,7 @@
// under the License.
using System.Buffers;
+using System.Runtime.CompilerServices;
namespace Apache.Fory;
@@ -142,15 +143,11 @@ public sealed class Fory
ByteWriter writer = _writeContext.Writer;
writer.Reset();
Serializer<T> serializer = _typeResolver.GetSerializer<T>();
- bool isNone = value is null;
- WriteHead(writer, isNone);
- if (!isNone)
- {
- _writeContext.ResetFor(writer);
- RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly;
- serializer.Write(_writeContext, value, refMode, true, false);
- _writeContext.RefWriter.Reset();
- }
+ WriteHead(writer);
+ _writeContext.ResetFor(writer);
+ RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly;
+ serializer.Write(_writeContext, value, refMode, true, false);
+ _writeContext.RefWriter.Reset();
return writer.ToArray();
}
@@ -181,7 +178,7 @@ public sealed class Fory
T value = DeserializeFromReader<T>(reader);
if (reader.Remaining != 0)
{
- throw new InvalidDataException($"unexpected trailing bytes after deserializing {typeof(T)}");
+ ThrowUnexpectedTrailingBytes<T>();
}
return value;
@@ -201,7 +198,7 @@ public sealed class Fory
T value = DeserializeFromReader<T>(reader);
if (reader.Remaining != 0)
{
- throw new InvalidDataException($"unexpected trailing bytes after deserializing {typeof(T)}");
+ ThrowUnexpectedTrailingBytes<T>();
}
return value;
@@ -228,46 +225,43 @@ public sealed class Fory
/// Writes the frame header for a payload.
/// </summary>
/// <param name="writer">Destination writer.</param>
- /// <param name="isNone">Whether the payload value is null.</param>
- internal void WriteHead(ByteWriter writer, bool isNone)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ internal void WriteHead(ByteWriter writer)
{
- byte bitmap = ForyHeaderFlag.IsXlang;
-
- if (isNone)
- {
- bitmap |= ForyHeaderFlag.IsNull;
- }
-
- writer.WriteUInt8(bitmap);
+ writer.WriteUInt8(ForyHeaderFlag.IsXlang);
}
/// <summary>
/// Reads and validates the frame header.
/// </summary>
/// <param name="reader">Source reader.</param>
- /// <returns><c>true</c> if the payload value is null; otherwise <c>false</c>.</returns>
/// <exception cref="InvalidDataException">Thrown when the peer xlang bitmap does not match this runtime mode.</exception>
- internal bool ReadHead(ByteReader reader)
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ internal void ReadHead(ByteReader reader)
{
byte bitmap = reader.ReadUInt8();
- bool peerIsXlang = (bitmap & ForyHeaderFlag.IsXlang) != 0;
- if (!peerIsXlang)
+ if (bitmap == ForyHeaderFlag.IsXlang)
{
- throw new InvalidDataException("xlang bitmap mismatch");
+ return;
}
-
- return (bitmap & ForyHeaderFlag.IsNull) != 0;
+ ThrowInvalidRootHeader(bitmap);
}
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static void ThrowUnexpectedTrailingBytes<T>() =>
+ throw new InvalidDataException($"unexpected trailing bytes after deserializing {typeof(T)}");
+
+ [MethodImpl(MethodImplOptions.NoInlining)]
+ private static void ThrowInvalidRootHeader(byte bitmap) =>
+ throw new InvalidDataException((bitmap & ForyHeaderFlag.IsXlang) == 0
+ ? "xlang bitmap mismatch"
+ : $"unsupported root header bitmap 0x{bitmap:X2}");
+
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
private T DeserializeFromReader<T>(ByteReader reader)
{
- bool isNone = ReadHead(reader);
+ ReadHead(reader);
Serializer<T> serializer = _typeResolver.GetSerializer<T>();
- if (isNone)
- {
- return serializer.DefaultValue;
- }
-
ReadContext readContext = _readContext;
readContext.ResetFor(reader);
RefMode refMode = Config.TrackRef ? RefMode.Tracking : RefMode.NullOnly;
diff --git a/csharp/src/Fory/ForyFlags.cs b/csharp/src/Fory/ForyFlags.cs
index 1a732c1d4..6f8613870 100644
--- a/csharp/src/Fory/ForyFlags.cs
+++ b/csharp/src/Fory/ForyFlags.cs
@@ -47,7 +47,7 @@ internal static class RefModeExtensions
internal static class ForyHeaderFlag
{
- public const byte IsNull = 0x01;
- public const byte IsXlang = 0x02;
- public const byte IsOutOfBand = 0x04;
+ public const byte IsXlang = 0x01;
+ public const byte IsOutOfBand = 0x02;
+ public const byte KnownMask = IsXlang | IsOutOfBand;
}
diff --git a/csharp/src/Fory/MetaString.cs b/csharp/src/Fory/MetaString.cs
index 34d3f07b7..df49932fd 100644
--- a/csharp/src/Fory/MetaString.cs
+++ b/csharp/src/Fory/MetaString.cs
@@ -430,6 +430,8 @@ internal sealed class MetaStringDecoder
{
string value = encoding switch
{
+ // C# intentionally preserves platform UTF-8 replacement behavior; Rust is the runtime
+ // that provides checked UTF-8 string reads by default.
MetaStringEncoding.Utf8 => Encoding.UTF8.GetString(bytes),
MetaStringEncoding.LowerSpecial => DecodeGeneric(bytes, 5, UnmapLowerSpecial),
MetaStringEncoding.LowerUpperDigitSpecial => DecodeGeneric(bytes, 6, UnmapLowerUpperDigitSpecial),
diff --git a/csharp/src/Fory/ReadContext.cs b/csharp/src/Fory/ReadContext.cs
index 53e23e2ee..6444500c4 100644
--- a/csharp/src/Fory/ReadContext.cs
+++ b/csharp/src/Fory/ReadContext.cs
@@ -21,14 +21,12 @@ public sealed class ReadContext
{
private const int MaxParsedTypeMetaEntries = 8192;
- private readonly record struct CachedTypeMetaEntry(TypeMeta TypeMeta, int SkipBytesAfterHeader);
-
private readonly ReusableArray<TypeMeta> _readTypeMetas = new();
- private readonly Dictionary<ulong, CachedTypeMetaEntry> _cachedTypeMetasByHeader = [];
+ private readonly Dictionary<ulong, TypeMeta> _cachedTypeMetasByHeader = [];
private TypeMeta? _firstReadTypeMeta;
private bool _hasFirstReadTypeMeta;
private ulong _lastMetaHeader;
- private CachedTypeMetaEntry _lastTypeMeta;
+ private TypeMeta? _lastTypeMeta;
private bool _hasLastMetaHeader;
private readonly List<MetaString> _readMetaStrings = [];
@@ -135,40 +133,46 @@ public sealed class ReadContext
$"type meta index gap: index={index}, count={_readTypeMetas.Count + 1}");
}
- internal bool TryGetCachedReadTypeMeta(ulong header, out TypeMeta typeMeta, out int skipBytesAfterHeader)
+ internal bool TryGetCachedReadTypeMeta(ulong header, out TypeMeta typeMeta)
{
- if (_hasLastMetaHeader && _lastMetaHeader == header)
+ if (_hasLastMetaHeader && _lastMetaHeader == header && _lastTypeMeta is not null)
{
- typeMeta = _lastTypeMeta.TypeMeta;
- skipBytesAfterHeader = _lastTypeMeta.SkipBytesAfterHeader;
+ typeMeta = _lastTypeMeta;
return true;
}
- if (_cachedTypeMetasByHeader.TryGetValue(header, out CachedTypeMetaEntry cached))
+ if (_cachedTypeMetasByHeader.TryGetValue(header, out TypeMeta? cached) && cached is not null)
{
_lastMetaHeader = header;
_lastTypeMeta = cached;
_hasLastMetaHeader = true;
- typeMeta = cached.TypeMeta;
- skipBytesAfterHeader = cached.SkipBytesAfterHeader;
+ typeMeta = cached;
return true;
}
typeMeta = null!;
- skipBytesAfterHeader = 0;
return false;
}
- internal void CacheReadTypeMeta(ulong header, TypeMeta typeMeta, int skipBytesAfterHeader)
+ internal void CacheReadTypeMeta(ulong header, TypeMeta typeMeta)
{
- CachedTypeMetaEntry cached = new(typeMeta, skipBytesAfterHeader);
- _lastMetaHeader = header;
- _lastTypeMeta = cached;
- _hasLastMetaHeader = true;
- if (_cachedTypeMetasByHeader.Count < MaxParsedTypeMetaEntries)
+ if (_cachedTypeMetasByHeader.TryGetValue(header, out TypeMeta? existing) && existing is not null)
{
- _cachedTypeMetasByHeader.TryAdd(header, cached);
+ _lastMetaHeader = header;
+ _lastTypeMeta = existing;
+ _hasLastMetaHeader = true;
+ return;
}
+
+ if (_cachedTypeMetasByHeader.Count >= MaxParsedTypeMetaEntries)
+ {
+ return;
+ }
+
+ _lastMetaHeader = header;
+ _lastTypeMeta = typeMeta;
+ _hasLastMetaHeader = true;
+ _cachedTypeMetasByHeader.TryAdd(header, typeMeta);
}
internal MetaString? GetReadMetaString(int index)
@@ -198,20 +202,20 @@ public sealed class ReadContext
}
ulong header = Reader.ReadUInt64();
- if (TryGetCachedReadTypeMeta(header, out TypeMeta cachedTypeMeta, out int skipBytesAfterHeader))
+ if (TryGetCachedReadTypeMeta(header, out TypeMeta cachedTypeMeta))
{
- Reader.Skip(skipBytesAfterHeader);
+ // Header-cache hits intentionally skip without rehashing. Entries reach this cache only
+ // after a successful TypeMeta parse and 52-bit body-hash validation. The current body
+ // size still comes from the current header bytes, not from the cached TypeMeta.
+ TypeMeta.SkipBody(Reader, header);
StoreReadTypeMeta(cachedTypeMeta, index);
return cachedTypeMeta;
}
- int headerStartCursor = Reader.Cursor - sizeof(ulong);
Reader.MoveBack(sizeof(ulong));
TypeMeta typeMeta = TypeMeta.Decode(Reader);
- int consumedTypeMetaBytes = Reader.Cursor - headerStartCursor;
- int parsedSkipBytesAfterHeader = consumedTypeMetaBytes - sizeof(ulong);
StoreReadTypeMeta(typeMeta, index);
- CacheReadTypeMeta(header, typeMeta, parsedSkipBytesAfterHeader);
+ CacheReadTypeMeta(header, typeMeta);
return typeMeta;
}
diff --git a/csharp/src/Fory/StringSerializer.cs b/csharp/src/Fory/StringSerializer.cs
index 78a822eb1..c176348c8 100644
--- a/csharp/src/Fory/StringSerializer.cs
+++ b/csharp/src/Fory/StringSerializer.cs
@@ -23,7 +23,6 @@ public sealed class StringSerializer : Serializer<string>
{
private const int MaxVarUInt36SmallBytes = 6;
-
public override string DefaultValue => null!;
public override void WriteData(WriteContext context, in string value, bool hasGenerics)
@@ -65,6 +64,8 @@ public sealed class StringSerializer : Serializer<string>
ReadOnlySpan<byte> bytes = context.Reader.ReadSpan(byteLength);
return encoding switch
{
+ // C# intentionally preserves platform UTF-8 replacement behavior; Rust is the runtime
+ // that provides checked UTF-8 string reads by default.
(ulong)ForyStringEncoding.Utf8 => Encoding.UTF8.GetString(bytes),
(ulong)ForyStringEncoding.Latin1 => DecodeLatin1(bytes),
(ulong)ForyStringEncoding.Utf16 => DecodeUtf16(bytes),
diff --git a/csharp/src/Fory/TypeInfo.cs b/csharp/src/Fory/TypeInfo.cs
index e6a9ecc32..95c47ec86 100644
--- a/csharp/src/Fory/TypeInfo.cs
+++ b/csharp/src/Fory/TypeInfo.cs
@@ -712,7 +712,6 @@ public sealed class TypeInfo
compatible: true,
Evolving);
IReadOnlyList<TypeMetaFieldInfo> fields = TypeMetaFields(trackRef);
- bool hasFieldsMeta = fields.Count > 0;
TypeMeta typeMeta;
if (RegisterByName)
{
@@ -727,8 +726,7 @@ public sealed class TypeInfo
NamespaceName.Value,
TypeName.Value,
true,
- fields,
- hasFieldsMeta);
+ fields);
}
else
{
@@ -743,13 +741,12 @@ public sealed class TypeInfo
MetaString.Empty('.', '_'),
MetaString.Empty('$', '_'),
false,
- fields,
- hasFieldsMeta);
+ fields);
}
byte[] encoded = typeMeta.Encode();
ulong header = BinaryPrimitives.ReadUInt64LittleEndian(encoded);
- ulong headerHash = header >> (int)(64 - TypeMetaConstants.TypeMetaNumHashBits);
+ ulong headerHash = header >> TypeMetaConstants.TypeMetaHashShift;
return new TypeMetaCacheEntry(typeMeta, encoded, headerHash);
}
}
diff --git a/csharp/src/Fory/TypeMeta.cs b/csharp/src/Fory/TypeMeta.cs
index 4d8415260..9073b5f1f 100644
--- a/csharp/src/Fory/TypeMeta.cs
+++ b/csharp/src/Fory/TypeMeta.cs
@@ -19,14 +19,18 @@ namespace Apache.Fory;
internal static class TypeMetaConstants
{
+ // 8 size bits + 1 compression bit + 3 reserved bits.
+ public const int TypeMetaHashShift = 12;
+ public const ulong TypeMetaHashMask = ulong.MaxValue << TypeMetaHashShift;
public const int SmallNumFieldsThreshold = 0b1_1111;
public const byte RegisterByNameFlag = 0b10_0000;
+ public const byte CompatibleFlag = 0b0100_0000;
+ public const byte StructFlag = 0b1000_0000;
public const int FieldNameSizeThreshold = 0b1111;
public const int BigNameThreshold = 0b11_1111;
- public const ulong TypeMetaHasFieldsMetaFlag = 1UL << 8;
- public const ulong TypeMetaCompressedFlag = 1UL << 9;
+ public const ulong TypeMetaCompressedFlag = 1UL << 8;
+ public const ulong TypeMetaReservedFlags = 0b111UL << 9;
public const ulong TypeMetaSizeMask = 0xFF;
- public const ulong TypeMetaNumHashBits = 50;
public const ulong TypeMetaHashSeed = 47;
public const uint NoUserTypeId = uint.MaxValue;
}
@@ -395,7 +399,6 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
MetaString typeName,
bool registerByName,
IReadOnlyList<TypeMetaFieldInfo> fields,
- bool hasFieldsMeta = true,
bool compressed = false,
ulong headerHash = 0)
{
@@ -425,7 +428,6 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
TypeName = typeName;
RegisterByName = registerByName;
Fields = fields;
- HasFieldsMeta = hasFieldsMeta;
Compressed = compressed;
HeaderHash = headerHash;
}
@@ -442,8 +444,6 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
public IReadOnlyList<TypeMetaFieldInfo> Fields { get; }
- public bool HasFieldsMeta { get; }
-
public bool Compressed { get; }
public ulong HeaderHash { get; }
@@ -467,22 +467,7 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
}
byte[] body = EncodeBody();
- (ulong bodyHash, _) = MurmurHash3.X64_128(body, TypeMetaConstants.TypeMetaHashSeed);
- ulong shifted = bodyHash << (int)(64 - TypeMetaConstants.TypeMetaNumHashBits);
- long signed = unchecked((long)shifted);
- long absSigned = signed == long.MinValue ? signed : Math.Abs(signed);
-
- ulong header = unchecked((ulong)absSigned);
- if (HasFieldsMeta)
- {
- header |= TypeMetaConstants.TypeMetaHasFieldsMetaFlag;
- }
-
- if (Compressed)
- {
- header |= TypeMetaConstants.TypeMetaCompressedFlag;
- }
-
+ ulong header = ComputeHeaderHashBits(body);
uint bodySize = (uint)Math.Min(body.Length, (int)TypeMetaConstants.TypeMetaSizeMask);
header |= bodySize;
ByteWriter writer = new(body.Length + 16);
@@ -504,43 +489,50 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
public static TypeMeta Decode(ByteReader reader)
{
ulong header = reader.ReadUInt64();
- bool compressed = (header & TypeMetaConstants.TypeMetaCompressedFlag) != 0;
- bool hasFieldsMeta = (header & TypeMetaConstants.TypeMetaHasFieldsMetaFlag) != 0;
- int metaSize = (int)(header & TypeMetaConstants.TypeMetaSizeMask);
- if (metaSize == (int)TypeMetaConstants.TypeMetaSizeMask)
- {
- metaSize += (int)reader.ReadVarUInt32();
- }
-
+ ValidateGlobalHeader(header);
+ int metaSize = ReadBodySize(reader, header);
byte[] encodedBody = reader.ReadBytes(metaSize);
- if (compressed)
- {
- throw new EncodingException("compressed TypeMeta is not supported yet");
- }
-
ByteReader bodyReader = new(encodedBody);
byte metaHeader = bodyReader.ReadUInt8();
- int numFields = metaHeader & TypeMetaConstants.SmallNumFieldsThreshold;
- if (numFields == TypeMetaConstants.SmallNumFieldsThreshold)
- {
- numFields += (int)bodyReader.ReadVarUInt32();
- }
-
- bool registerByName = (metaHeader & TypeMetaConstants.RegisterByNameFlag) != 0;
+ bool isStruct = (metaHeader & TypeMetaConstants.StructFlag) != 0;
+ int numFields = 0;
+ bool registerByName;
uint? typeId;
uint? userTypeId;
MetaString namespaceName;
MetaString typeName;
+ if (isStruct)
+ {
+ registerByName = (metaHeader & TypeMetaConstants.RegisterByNameFlag) != 0;
+ bool compatible = (metaHeader & TypeMetaConstants.CompatibleFlag) != 0;
+ typeId = (uint)(registerByName
+ ? compatible ? global::Apache.Fory.TypeId.NamedCompatibleStruct : global::Apache.Fory.TypeId.NamedStruct
+ : compatible ? global::Apache.Fory.TypeId.CompatibleStruct : global::Apache.Fory.TypeId.Struct);
+ numFields = metaHeader & TypeMetaConstants.SmallNumFieldsThreshold;
+ if (numFields == TypeMetaConstants.SmallNumFieldsThreshold)
+ {
+ numFields += (int)bodyReader.ReadVarUInt32();
+ }
+ }
+ else
+ {
+ if ((metaHeader & 0b0111_0000) != 0)
+ {
+ throw new InvalidDataException("invalid TypeMeta kind header");
+ }
+
+ typeId = NonStructTypeId(metaHeader & 0b1111);
+ registerByName = IsNamedKind(typeId.Value);
+ }
+
if (registerByName)
{
namespaceName = ReadName(bodyReader, MetaStringDecoder.Namespace, TypeMetaEncodings.NamespaceMetaStringEncodings);
typeName = ReadName(bodyReader, MetaStringDecoder.TypeName, TypeMetaEncodings.TypeNameMetaStringEncodings);
- typeId = null;
userTypeId = null;
}
else
{
- typeId = bodyReader.ReadUInt8();
userTypeId = bodyReader.ReadVarUInt32();
namespaceName = MetaString.Empty('.', '_');
typeName = MetaString.Empty('$', '_');
@@ -552,11 +544,17 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
fields.Add(TypeMetaFieldInfo.Read(bodyReader));
}
+ if (!isStruct && fields.Count != 0)
+ {
+ throw new InvalidDataException("non-struct TypeMeta cannot carry field metadata");
+ }
+
if (bodyReader.Remaining != 0)
{
throw new InvalidDataException("unexpected trailing bytes in TypeMeta body");
}
+ ValidateParsedTypeMetaHash(header, encodedBody);
return new TypeMeta(
typeId,
userTypeId,
@@ -564,9 +562,117 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
typeName,
registerByName,
fields,
- hasFieldsMeta,
- compressed,
- header >> (int)(64 - TypeMetaConstants.TypeMetaNumHashBits));
+ compressed: false,
+ header >> TypeMetaConstants.TypeMetaHashShift);
+ }
+
+ internal static void ValidateAndSkipBody(ByteReader reader, ulong header)
+ {
+ ValidateGlobalHeader(header);
+ int metaSize = ReadBodySize(reader, header);
+ ReadOnlySpan<byte> encodedBody = reader.ReadSpan(metaSize);
+ ValidateParsedTypeMetaHash(header, encodedBody);
+ }
+
+ private static void ValidateGlobalHeader(ulong header)
+ {
+ if ((header & TypeMetaConstants.TypeMetaReservedFlags) != 0)
+ {
+ throw new InvalidDataException("invalid TypeMeta global header");
+ }
+
+ if ((header & TypeMetaConstants.TypeMetaCompressedFlag) != 0)
+ {
+ throw new EncodingException("compressed TypeMeta is not supported yet");
+ }
+ }
+
+ private static int ReadBodySize(ByteReader reader, ulong header)
+ {
+ int metaSize = (int)(header & TypeMetaConstants.TypeMetaSizeMask);
+ if (metaSize == (int)TypeMetaConstants.TypeMetaSizeMask)
+ {
+ uint moreSize = reader.ReadVarUInt32();
+ if (moreSize > int.MaxValue - metaSize)
+ {
+ throw new InvalidDataException("invalid TypeMeta metadata size");
+ }
+
+ metaSize += (int)moreSize;
+ }
+
+ return metaSize;
+ }
+
+ internal static void SkipBody(ByteReader reader, ulong header)
+ {
+ reader.Skip(ReadBodySize(reader, header));
+ }
+
+ private static ulong ComputeHeaderHashBits(ReadOnlySpan<byte> body)
+ {
+ (ulong bodyHash, _) = MurmurHash3.X64_128(body, TypeMetaConstants.TypeMetaHashSeed);
+ ulong shifted = bodyHash << TypeMetaConstants.TypeMetaHashShift;
+ long signed = unchecked((long)shifted);
+ long absSigned = signed == long.MinValue ? signed : Math.Abs(signed);
+ return unchecked((ulong)absSigned) & TypeMetaConstants.TypeMetaHashMask;
+ }
+
+ private static void ValidateParsedTypeMetaHash(ulong header, ReadOnlySpan<byte> body)
+ {
+ ulong expectedHeaderHash = ComputeHeaderHashBits(body);
+ ulong actualHeaderHash = header & TypeMetaConstants.TypeMetaHashMask;
+ if (actualHeaderHash != expectedHeaderHash)
+ {
+ throw new InvalidDataException("TypeMeta metadata hash mismatch");
+ }
+ }
+
+ private static uint NonStructKindCode(uint typeId)
+ {
+ return (global::Apache.Fory.TypeId)typeId switch
+ {
+ global::Apache.Fory.TypeId.Enum => 0,
+ global::Apache.Fory.TypeId.NamedEnum => 1,
+ global::Apache.Fory.TypeId.Ext => 2,
+ global::Apache.Fory.TypeId.NamedExt => 3,
+ global::Apache.Fory.TypeId.TypedUnion => 4,
+ global::Apache.Fory.TypeId.NamedUnion => 5,
+ _ => throw new EncodingException($"unsupported TypeMeta kind {typeId}"),
+ };
+ }
+
+ private static uint NonStructTypeId(int kindCode)
+ {
+ return kindCode switch
+ {
+ 0 => (uint)global::Apache.Fory.TypeId.Enum,
+ 1 => (uint)global::Apache.Fory.TypeId.NamedEnum,
+ 2 => (uint)global::Apache.Fory.TypeId.Ext,
+ 3 => (uint)global::Apache.Fory.TypeId.NamedExt,
+ 4 => (uint)global::Apache.Fory.TypeId.TypedUnion,
+ 5 => (uint)global::Apache.Fory.TypeId.NamedUnion,
+ _ => throw new InvalidDataException($"unsupported TypeMeta kind code {kindCode}"),
+ };
+ }
+
+ private static bool IsStructKind(uint typeId)
+ {
+ return (global::Apache.Fory.TypeId)typeId is
+ global::Apache.Fory.TypeId.Struct or
+ global::Apache.Fory.TypeId.CompatibleStruct or
+ global::Apache.Fory.TypeId.NamedStruct or
+ global::Apache.Fory.TypeId.NamedCompatibleStruct;
+ }
+
+ private static bool IsNamedKind(uint typeId)
+ {
+ return (global::Apache.Fory.TypeId)typeId is
+ global::Apache.Fory.TypeId.NamedStruct or
+ global::Apache.Fory.TypeId.NamedCompatibleStruct or
+ global::Apache.Fory.TypeId.NamedEnum or
+ global::Apache.Fory.TypeId.NamedExt or
+ global::Apache.Fory.TypeId.NamedUnion;
}
/// <summary>
@@ -708,14 +814,40 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
private byte[] EncodeBody()
{
ByteWriter writer = new(128);
- byte metaHeader = (byte)Math.Min(Fields.Count, TypeMetaConstants.SmallNumFieldsThreshold);
- if (RegisterByName)
+ if (!TypeId.HasValue)
+ {
+ throw new EncodingException("type id is required");
+ }
+
+ bool isStruct = IsStructKind(TypeId.Value);
+ if (!isStruct && Fields.Count != 0)
+ {
+ throw new EncodingException("non-struct TypeMeta cannot carry field metadata");
+ }
+
+ byte metaHeader;
+ if (isStruct)
{
- metaHeader |= TypeMetaConstants.RegisterByNameFlag;
+ metaHeader = (byte)(TypeMetaConstants.StructFlag |
+ Math.Min(Fields.Count, TypeMetaConstants.SmallNumFieldsThreshold));
+ if (TypeId.Value is (uint)global::Apache.Fory.TypeId.CompatibleStruct or
+ (uint)global::Apache.Fory.TypeId.NamedCompatibleStruct)
+ {
+ metaHeader |= TypeMetaConstants.CompatibleFlag;
+ }
+
+ if (RegisterByName)
+ {
+ metaHeader |= TypeMetaConstants.RegisterByNameFlag;
+ }
+ }
+ else
+ {
+ metaHeader = (byte)NonStructKindCode(TypeId.Value);
}
writer.WriteUInt8(metaHeader);
- if (Fields.Count >= TypeMetaConstants.SmallNumFieldsThreshold)
+ if (isStruct && Fields.Count >= TypeMetaConstants.SmallNumFieldsThreshold)
{
writer.WriteVarUInt32((uint)(Fields.Count - TypeMetaConstants.SmallNumFieldsThreshold));
}
@@ -727,17 +859,11 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
}
else
{
- if (!TypeId.HasValue)
- {
- throw new EncodingException("type id is required in register-by-id mode");
- }
-
if (!UserTypeId.HasValue || UserTypeId == TypeMetaConstants.NoUserTypeId)
{
throw new EncodingException("user type id is required in register-by-id mode");
}
- writer.WriteUInt8(unchecked((byte)TypeId.Value));
writer.WriteVarUInt32(UserTypeId.Value);
}
@@ -811,7 +937,6 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
TypeName.Equals(other.TypeName) &&
RegisterByName == other.RegisterByName &&
Fields.SequenceEqual(other.Fields) &&
- HasFieldsMeta == other.HasFieldsMeta &&
Compressed == other.Compressed &&
HeaderHash == other.HeaderHash;
}
@@ -829,7 +954,6 @@ public sealed class TypeMeta : IEquatable<TypeMeta>
hc.Add(NamespaceName);
hc.Add(TypeName);
hc.Add(RegisterByName);
- hc.Add(HasFieldsMeta);
hc.Add(Compressed);
hc.Add(HeaderHash);
foreach (TypeMetaFieldInfo f in Fields)
diff --git a/csharp/src/Fory/TypeResolver.cs b/csharp/src/Fory/TypeResolver.cs
index 86b86e680..91a355630 100644
--- a/csharp/src/Fory/TypeResolver.cs
+++ b/csharp/src/Fory/TypeResolver.cs
@@ -937,6 +937,7 @@ public sealed class TypeResolver
private TypeInfo ResolveAnyTypeInfoFromMeta(TypeId wireTypeId, TypeMeta typeMeta, bool compatible)
{
+ ValidateTypeMetaWireType(typeMeta, wireTypeId);
TypeInfo typeInfo = typeMeta.RegisterByName
? RequireRegisteredTypeInfoByName(typeMeta.NamespaceName.Value, typeMeta.TypeName.Value)
: typeMeta.UserTypeId.HasValue
@@ -1233,6 +1234,8 @@ public sealed class TypeResolver
bool compatible,
TypeId actualWireTypeId)
{
+ ValidateTypeMetaWireType(remoteTypeMeta, actualWireTypeId);
+
if (remoteTypeMeta.RegisterByName)
{
if (!localInfo.RegisterByName || !localInfo.NamespaceName.HasValue || !localInfo.TypeName.HasValue)
@@ -1288,6 +1291,19 @@ public sealed class TypeResolver
}
}
+ private static void ValidateTypeMetaWireType(TypeMeta remoteTypeMeta, TypeId actualWireTypeId)
+ {
+ if (!remoteTypeMeta.TypeId.HasValue || !IsKnownTypeId(remoteTypeMeta.TypeId.Value))
+ {
+ throw new InvalidDataException("missing or unknown TypeMeta kind");
+ }
+
+ if ((TypeId)remoteTypeMeta.TypeId.Value != actualWireTypeId)
+ {
+ throw new TypeMismatchException((uint)actualWireTypeId, remoteTypeMeta.TypeId.Value);
+ }
+ }
+
private static void WriteMetaString(
WriteContext context,
MetaString value,
diff --git a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs
index 610b44fe1..b3faee5e7 100644
--- a/csharp/tests/Fory.Tests/ForyRuntimeTests.cs
+++ b/csharp/tests/Fory.Tests/ForyRuntimeTests.cs
@@ -833,7 +833,7 @@ public sealed class ForyRuntimeTests
byte[] data = fory.Serialize(value);
ByteReader reader = new(data);
- _ = fory.ReadHead(reader);
+ fory.ReadHead(reader);
_ = reader.ReadInt8();
_ = reader.ReadVarUInt32();
_ = reader.ReadVarUInt32();
@@ -1520,13 +1520,28 @@ public sealed class ForyRuntimeTests
writer.Register<OneStringField>(200);
byte[] payload = writer.Serialize(new OneStringField { F1 = "hello" });
- byte[] tamperedPayload = RewriteCompatibleTypeMetaTypeId(payload, (uint)TypeId.Map);
+ byte[] tamperedPayload = RewriteCompatibleTypeMetaTypeId(payload, (uint)TypeId.Struct);
ForyRuntime reader = ForyRuntime.Builder().Compatible(true).Build();
reader.Register<OneStringField>(200);
Assert.Throws<TypeMismatchException>(() => reader.Deserialize<OneStringField>(tamperedPayload));
}
+ [Fact]
+ public void CompatibleTypeMetaCacheMissValidatesBodyHashBeforeCaching()
+ {
+ ForyRuntime writer = ForyRuntime.Builder().Compatible(true).Build();
+ writer.Register<OneStringField>(200);
+ byte[] payload = writer.Serialize(new OneStringField { F1 = "hello" });
+ byte[] tamperedPayload = CorruptCompatibleTypeMetaBody(payload);
+
+ ForyRuntime reader = ForyRuntime.Builder().Compatible(true).Build();
+ reader.Register<OneStringField>(200);
+ InvalidDataException exception =
+ Assert.Throws<InvalidDataException>(() => reader.Deserialize<OneStringField>(tamperedPayload));
+ Assert.Contains("TypeMeta metadata hash mismatch", exception.Message, StringComparison.Ordinal);
+ }
+
[Fact]
public void TypeMetaAssignFieldIdsPrefersIdAndFallsBackToName()
{
@@ -1653,21 +1668,7 @@ public sealed class ForyRuntimeTests
private static byte[] RewriteCompatibleTypeMetaTypeId(byte[] payload, uint embeddedTypeId)
{
- ByteReader reader = new(payload);
- _ = reader.ReadUInt8(); // frame header bitmap
-
- sbyte refFlag = reader.ReadInt8();
- Assert.Equal((sbyte)RefFlag.NotNullValue, refFlag);
-
- uint wireTypeId = reader.ReadUInt8();
- Assert.Equal((uint)TypeId.CompatibleStruct, wireTypeId);
-
- uint typeMetaIndexMarker = reader.ReadVarUInt32();
- Assert.Equal(0u, typeMetaIndexMarker & 1u);
-
- int typeMetaStart = reader.Cursor;
- TypeMeta originalTypeMeta = TypeMeta.Decode(reader);
- int typeMetaEnd = reader.Cursor;
+ (int typeMetaStart, int typeMetaEnd, TypeMeta originalTypeMeta) = ReadCompatibleTypeMetaRange(payload);
TypeMeta rewrittenTypeMeta = new(
embeddedTypeId,
@@ -1676,7 +1677,6 @@ public sealed class ForyRuntimeTests
originalTypeMeta.TypeName,
originalTypeMeta.RegisterByName,
originalTypeMeta.Fields,
- originalTypeMeta.HasFieldsMeta,
originalTypeMeta.Compressed);
byte[] rewrittenTypeMetaBytes = rewrittenTypeMeta.Encode();
@@ -1687,6 +1687,35 @@ public sealed class ForyRuntimeTests
return rewrittenPayload;
}
+ private static byte[] CorruptCompatibleTypeMetaBody(byte[] payload)
+ {
+ (int typeMetaStart, int typeMetaEnd, _) = ReadCompatibleTypeMetaRange(payload);
+ Assert.True(typeMetaEnd > typeMetaStart + sizeof(ulong));
+ byte[] malformed = (byte[])payload.Clone();
+ malformed[typeMetaEnd - 1] ^= 1;
+ return malformed;
+ }
+
+ private static (int TypeMetaStart, int TypeMetaEnd, TypeMeta TypeMeta) ReadCompatibleTypeMetaRange(byte[] payload)
+ {
+ ByteReader reader = new(payload);
+ _ = reader.ReadUInt8(); // frame header bitmap
+
+ sbyte refFlag = reader.ReadInt8();
+ Assert.Equal((sbyte)RefFlag.NotNullValue, refFlag);
+
+ uint wireTypeId = reader.ReadUInt8();
+ Assert.Equal((uint)TypeId.CompatibleStruct, wireTypeId);
+
+ uint typeMetaIndexMarker = reader.ReadVarUInt32();
+ Assert.Equal(0u, typeMetaIndexMarker & 1u);
+
+ int typeMetaStart = reader.Cursor;
+ TypeMeta originalTypeMeta = TypeMeta.Decode(reader);
+ int typeMetaEnd = reader.Cursor;
+ return (typeMetaStart, typeMetaEnd, originalTypeMeta);
+ }
+
private static (ulong Encoding, string Decoded) WriteAndReadString(string value)
{
ByteWriter writer = new();
diff --git a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs
index a74b21fd3..3b3011a3f 100644
--- a/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs
+++ b/csharp/tests/Fory.Tests/RuntimeEdgeCaseTests.cs
@@ -160,7 +160,7 @@ public sealed class RuntimeEdgeCaseTests
byte[] payload = fory.Serialize(TimeSpan.FromSeconds(1) + TimeSpan.FromTicks(3));
ByteReader reader = new(payload);
- Assert.False(fory.ReadHead(reader));
+ fory.ReadHead(reader);
Assert.Equal((sbyte)RefFlag.NotNullValue, reader.ReadInt8());
Assert.Equal((uint)TypeId.Duration, reader.ReadUInt8());
Assert.Equal(1L, reader.ReadVarInt64());
@@ -175,7 +175,7 @@ public sealed class RuntimeEdgeCaseTests
byte[] payload = fory.Serialize(new DateOnly(2021, 11, 23));
ByteReader reader = new(payload);
- Assert.False(fory.ReadHead(reader));
+ fory.ReadHead(reader);
Assert.Equal((sbyte)RefFlag.NotNullValue, reader.ReadInt8());
Assert.Equal((uint)TypeId.Date, reader.ReadUInt8());
Assert.Equal(18_954L, reader.ReadVarInt64());
@@ -252,7 +252,7 @@ public sealed class RuntimeEdgeCaseTests
byte[] payload = fory.Serialize(new ForyDecimal(BigInteger.Zero, 2));
ByteReader reader = new(payload);
- Assert.False(fory.ReadHead(reader));
+ fory.ReadHead(reader);
Assert.Equal((sbyte)RefFlag.NotNullValue, reader.ReadInt8());
Assert.Equal((uint)TypeId.Decimal, reader.ReadUInt8());
Assert.Equal(2, reader.ReadVarInt32());
@@ -261,7 +261,7 @@ public sealed class RuntimeEdgeCaseTests
payload = fory.Serialize(new ForyDecimal(BigInteger.Parse("9223372036854775808"), 0));
reader.Reset(payload);
- Assert.False(fory.ReadHead(reader));
+ fory.ReadHead(reader);
Assert.Equal((sbyte)RefFlag.NotNullValue, reader.ReadInt8());
Assert.Equal((uint)TypeId.Decimal, reader.ReadUInt8());
Assert.Equal(0, reader.ReadVarInt32());
@@ -275,7 +275,7 @@ public sealed class RuntimeEdgeCaseTests
{
ForyRuntime fory = ForyRuntime.Builder().Build();
ByteWriter writer = new();
- fory.WriteHead(writer, isNone: false);
+ fory.WriteHead(writer);
writer.WriteInt8((sbyte)RefFlag.NotNullValue);
writer.WriteUInt8((byte)TypeId.Decimal);
writer.WriteVarInt32(0);
@@ -283,7 +283,7 @@ public sealed class RuntimeEdgeCaseTests
_ = Assert.Throws<InvalidDataException>(() => fory.Deserialize<ForyDecimal>(writer.ToArray()));
writer.Reset();
- fory.WriteHead(writer, isNone: false);
+ fory.WriteHead(writer);
writer.WriteInt8((sbyte)RefFlag.NotNullValue);
writer.WriteUInt8((byte)TypeId.Decimal);
writer.WriteVarInt32(0);
@@ -302,7 +302,7 @@ public sealed class RuntimeEdgeCaseTests
byte[] payload = fory.Serialize(DateTimeOffset.FromUnixTimeMilliseconds(-1));
ByteReader reader = new(payload);
- Assert.False(fory.ReadHead(reader));
+ fory.ReadHead(reader);
Assert.Equal((sbyte)RefFlag.NotNullValue, reader.ReadInt8());
Assert.Equal((uint)TypeId.Timestamp, reader.ReadUInt8());
Assert.Equal(-1L, reader.ReadInt64());
@@ -407,6 +407,83 @@ public sealed class RuntimeEdgeCaseTests
Assert.Contains("xlang bitmap mismatch", exception.Message, StringComparison.Ordinal);
}
+ [Fact]
+ public void SerializeNullRootUsesRefMeta()
+ {
+ ForyRuntime fory = ForyRuntime.Builder().Build();
+ byte[] payload = fory.Serialize<string?>(null);
+
+ Assert.Equal(ForyHeaderFlag.IsXlang, payload[0]);
+ Assert.Equal(unchecked((byte)(sbyte)RefFlag.Null), payload[1]);
+ Assert.Null(fory.Deserialize<string?>(payload));
+ }
+
+ [Fact]
+ public void DeserializeRejectsUnsupportedRootHeaderBits()
+ {
+ ForyRuntime fory = ForyRuntime.Builder().Build();
+ byte[] payload = fory.Serialize(123);
+
+ foreach (byte bitmap in new[] { (byte)0x03, (byte)0x05, (byte)0x81 })
+ {
+ byte[] invalidPayload = [.. payload];
+ invalidPayload[0] = bitmap;
+
+ InvalidDataException exception =
+ Assert.Throws<InvalidDataException>(() => fory.Deserialize<int>(invalidPayload));
+ Assert.Contains("unsupported root header bitmap", exception.Message, StringComparison.Ordinal);
+ }
+ }
+
+ [Fact]
+ public void TypeMetaHeaderCacheStopsPublishingAtCapacity()
+ {
+ ReadContext context = new(new ByteReader(Array.Empty<byte>()), new TypeResolver(), trackRef: false);
+ TypeMeta typeMeta = new(
+ (uint)TypeId.Struct,
+ 901,
+ MetaString.Empty('.', '_'),
+ MetaString.Empty('$', '_'),
+ registerByName: false,
+ []);
+
+ for (ulong header = 1; header <= 8192; header++)
+ {
+ context.CacheReadTypeMeta(header, typeMeta);
+ }
+
+ Assert.True(context.TryGetCachedReadTypeMeta(8192, out _));
+ context.CacheReadTypeMeta(8193, typeMeta);
+
+ Assert.False(context.TryGetCachedReadTypeMeta(8193, out _));
+ }
+
+ [Fact]
+ public void TypeMetaHeaderCacheHitSkipsCurrentBodySize()
+ {
+ const ulong header = 0xffUL;
+ TypeMeta typeMeta = new(
+ (uint)TypeId.Struct,
+ 902,
+ MetaString.Empty('.', '_'),
+ MetaString.Empty('$', '_'),
+ registerByName: false,
+ []);
+
+ ByteWriter writer = new();
+ writer.WriteVarUInt32(0);
+ writer.WriteUInt64(header);
+ writer.WriteVarUInt32(0);
+ writer.WriteBytes(new byte[0xff]);
+ writer.WriteUInt8(0x7b);
+
+ ReadContext context = new(new ByteReader(writer.ToArray()), new TypeResolver(), trackRef: false);
+ context.CacheReadTypeMeta(header, typeMeta);
+
+ Assert.Same(typeMeta, context.ReadTypeMeta());
+ Assert.Equal(0x7b, context.Reader.ReadUInt8());
+ }
+
[Fact]
public void DynamicAnyRejectsUnknownUserTypeId()
{
diff --git a/dart/packages/fory/lib/src/context/read_context.dart b/dart/packages/fory/lib/src/context/read_context.dart
index ebec1f320..27dd038db 100644
--- a/dart/packages/fory/lib/src/context/read_context.dart
+++ b/dart/packages/fory/lib/src/context/read_context.dart
@@ -233,9 +233,38 @@ final class ReadContext {
/// Reads a root value using Ref semantics and expected root type [T].
Object? readRefAs<T>() {
- return _readRefWithResolved(
- (resolved) => _typeResolver.resolveExpectedRootWireType<T>(resolved),
+ final flag = _refReader.tryPreserveRefId(_buffer);
+ final preservedRefId = flag >= RefWriter.refValueFlag ? flag : null;
+ if (flag == RefWriter.nullFlag) {
+ return null;
+ }
+ if (flag == RefWriter.refFlag) {
+ return _refReader.getReadRef();
+ }
+ final resolved = _typeResolver.resolveExpectedRootWireType<T>(
+ _readTypeMeta(),
);
+ final rootPreservedRefId = preservedRefId == null &&
+ flag == RefWriter.notNullValueFlag &&
+ _depth == 0 &&
+ resolved.supportsRef
+ ? _refReader.preserveRefId()
+ : null;
+ final value = readResolvedValue(
+ resolved,
+ null,
+ hasPreservedRef: preservedRefId != null || rootPreservedRefId != null,
+ );
+ if (preservedRefId != null &&
+ resolved.supportsRef &&
+ _refReader.readRefAt(preservedRefId) == null) {
+ _refReader.setReadRef(preservedRefId, value);
+ }
+ if (rootPreservedRefId != null &&
+ _refReader.readRefAt(rootPreservedRefId) == null) {
+ _refReader.setReadRef(rootPreservedRefId, value);
+ }
+ return value;
}
Object? _readRefWithResolved(TypeInfo Function(TypeInfo) resolveRootType) {
diff --git a/dart/packages/fory/lib/src/fory.dart b/dart/packages/fory/lib/src/fory.dart
index 80d819048..96d917538 100644
--- a/dart/packages/fory/lib/src/fory.dart
+++ b/dart/packages/fory/lib/src/fory.dart
@@ -39,9 +39,9 @@ import 'package:fory/src/serializer/serializer.dart';
///
/// The Dart runtime only supports xlang payloads.
final class Fory {
- static const int _nullHeaderFlag = 0x01;
- static const int _xlangHeaderFlag = 0x02;
- static const int _outOfBandHeaderFlag = 0x04;
+ static const int _xlangHeaderFlag = 0x01;
+ static const int _outOfBandHeaderFlag = 0x02;
+ static const int _knownHeaderFlags = _xlangHeaderFlag | _outOfBandHeaderFlag;
late final Buffer _readBuffer;
late final Buffer _writeBuffer;
@@ -124,11 +124,11 @@ final class Fory {
buffer.clear();
_writeContext.prepare(buffer, trackRef: trackRef);
try {
+ buffer.writeUint8(_xlangHeaderFlag);
if (value == null) {
- buffer.writeUint8(_nullHeaderFlag);
+ _writeContext.writeRootValue(null, trackRef: trackRef);
return;
}
- buffer.writeUint8(_xlangHeaderFlag);
_writeContext.writeRootValue(value, trackRef: trackRef);
} finally {
_writeContext.reset();
@@ -183,18 +183,8 @@ final class Fory {
_readContext.prepare(buffer);
try {
final header = buffer.readUint8();
- if ((header & _outOfBandHeaderFlag) != 0) {
- throw StateError(
- 'Out-of-band buffers are not supported by the Dart runtime.',
- );
- }
- if ((header & _nullHeaderFlag) != 0) {
- return null as T;
- }
- if ((header & _xlangHeaderFlag) == 0) {
- throw StateError(
- 'Only xlang payloads are supported by the Dart runtime.',
- );
+ if (header != _xlangHeaderFlag) {
+ _throwInvalidRootHeader(header);
}
final value = _readContext.readRefAs<T>();
if (value is T) {
@@ -208,6 +198,23 @@ final class Fory {
}
}
+ @pragma('vm:never-inline')
+ Never _throwInvalidRootHeader(int header) {
+ if ((header & ~_knownHeaderFlags) != 0) {
+ throw StateError(
+ 'Unsupported root header bitmap 0x${header.toRadixString(16)}.',
+ );
+ }
+ if ((header & _outOfBandHeaderFlag) != 0) {
+ throw StateError(
+ 'Out-of-band buffers are not supported by the Dart runtime.',
+ );
+ }
+ throw StateError(
+ 'Only xlang payloads are supported by the Dart runtime.',
+ );
+ }
+
/// Registers a generated type.
///
/// Exactly one registration mode is required:
diff --git a/dart/packages/fory/lib/src/meta/meta_string.dart b/dart/packages/fory/lib/src/meta/meta_string.dart
index abd60d517..7089d0071 100644
--- a/dart/packages/fory/lib/src/meta/meta_string.dart
+++ b/dart/packages/fory/lib/src/meta/meta_string.dart
@@ -32,6 +32,8 @@ const int metaStringAllToLowerSpecialEncoding = 4;
const int metaStringSmallThreshold = 16;
const int typeDefSmallFieldCountThreshold = 0x1f;
const int typeDefRegisterByNameFlag = 0x20;
+const int typeDefCompatibleFlag = 0x40;
+const int typeDefStructFlag = 0x80;
const int typeDefBigFieldNameThreshold = 0x0f;
const int typeDefBigNameThreshold = 0x3f;
diff --git a/dart/packages/fory/lib/src/meta/type_meta.dart b/dart/packages/fory/lib/src/meta/type_meta.dart
index dc8a2df5e..8b32f199f 100644
--- a/dart/packages/fory/lib/src/meta/type_meta.dart
+++ b/dart/packages/fory/lib/src/meta/type_meta.dart
@@ -18,6 +18,7 @@
*/
import 'dart:collection';
+import 'dart:typed_data';
import 'package:fory/src/memory/buffer.dart';
import 'package:fory/src/config.dart';
@@ -25,6 +26,7 @@ import 'package:fory/src/meta/meta_string.dart';
import 'package:fory/src/meta/type_ids.dart';
import 'package:fory/src/resolver/type_resolver.dart';
import 'package:fory/src/types/int64.dart';
+import 'package:fory/src/util/hash_util.dart';
/// Wire-level type metadata for one value.
final class WireTypeMeta {
@@ -53,10 +55,24 @@ final class WireTypeMeta {
}
final class TypeHeader {
+ static const int _compressMetaFlag = 1 << 8;
+ static const int _reservedMetaFlags = 0x0e00;
+ static const int _hashLow32Mask = 0xfffff000;
+
final Int64 value;
const TypeHeader(this.value);
+ @pragma('vm:prefer-inline')
+ void validateGlobal() {
+ if ((value.low32 & _reservedMetaFlags) != 0) {
+ throw StateError('Invalid TypeDef global header.');
+ }
+ if ((value.low32 & _compressMetaFlag) != 0) {
+ throw StateError('Compressed TypeDef metadata is not supported.');
+ }
+ }
+
@pragma('vm:prefer-inline')
int readMetaSize(Buffer buffer) {
final lowBits = value.low32 & 0xff;
@@ -70,6 +86,15 @@ final class TypeHeader {
void skipRemaining(Buffer buffer) {
buffer.skip(readMetaSize(buffer));
}
+
+ @pragma('vm:prefer-inline')
+ void validateBodyHash(Uint8List body) {
+ final expected = typeDefHeader(body);
+ if (value.high32Unsigned != expected.high32Unsigned ||
+ (value.low32 & _hashLow32Mask) != (expected.low32 & _hashLow32Mask)) {
+ throw StateError('Invalid TypeDef metadata hash.');
+ }
+ }
}
final class ParsedTypeMetaCache {
@@ -95,9 +120,17 @@ final class ParsedTypeMetaCache {
@pragma('vm:prefer-inline')
void remember(TypeHeader header, TypeInfo resolved) {
- if (!_entries.containsKey(header.value) && _entries.length >= maxEntries) {
- _entries.remove(_entries.keys.first);
+ final cached = _entries[header.value];
+ if (cached != null) {
+ _entries[header.value] = resolved;
+ _lastHeader = header.value;
+ _lastResolved = resolved;
+ return;
}
+ if (_entries.length >= maxEntries) {
+ return;
+ }
+
_entries[header.value] = resolved;
_lastHeader = header.value;
_lastResolved = resolved;
@@ -109,7 +142,7 @@ final class WireTypeMetaEncoder {
const WireTypeMetaEncoder();
WireTypeMeta typeMetaFor(Config config, TypeInfo resolvedType) {
- final wireTypeId = _wireTypeIdFor(config, resolvedType);
+ final wireTypeId = wireTypeIdFor(config, resolvedType);
final writesTypeDef = wireTypeId == TypeIds.compatibleStruct ||
wireTypeId == TypeIds.namedCompatibleStruct ||
(config.compatible &&
@@ -146,7 +179,8 @@ final class WireTypeMetaEncoder {
}
}
- int _wireTypeIdFor(Config config, TypeInfo resolvedType) {
+ @pragma('vm:prefer-inline')
+ int wireTypeIdFor(Config config, TypeInfo resolvedType) {
switch (resolvedType.kind) {
case RegistrationKind.builtin:
return resolvedType.typeId;
diff --git a/dart/packages/fory/lib/src/resolver/type_resolver.dart b/dart/packages/fory/lib/src/resolver/type_resolver.dart
index cc7a87c0f..40be46571 100644
--- a/dart/packages/fory/lib/src/resolver/type_resolver.dart
+++ b/dart/packages/fory/lib/src/resolver/type_resolver.dart
@@ -55,6 +55,57 @@ import 'package:fory/src/util/hash_util.dart';
enum RegistrationKind { builtin, struct, enumType, ext, union }
+bool _isStructTypeDefKind(int typeId) =>
+ typeId == TypeIds.struct ||
+ typeId == TypeIds.compatibleStruct ||
+ typeId == TypeIds.namedStruct ||
+ typeId == TypeIds.namedCompatibleStruct;
+
+bool _isNamedTypeDefKind(int typeId) =>
+ typeId == TypeIds.namedStruct ||
+ typeId == TypeIds.namedCompatibleStruct ||
+ typeId == TypeIds.namedEnum ||
+ typeId == TypeIds.namedExt ||
+ typeId == TypeIds.namedUnion;
+
+int _nonStructTypeDefKindCode(int typeId) {
+ switch (typeId) {
+ case TypeIds.enumById:
+ return 0;
+ case TypeIds.namedEnum:
+ return 1;
+ case TypeIds.ext:
+ return 2;
+ case TypeIds.namedExt:
+ return 3;
+ case TypeIds.typedUnion:
+ return 4;
+ case TypeIds.namedUnion:
+ return 5;
+ default:
+ throw StateError('Unsupported TypeDef kind $typeId.');
+ }
+}
+
+int _nonStructTypeDefTypeId(int kindCode) {
+ switch (kindCode) {
+ case 0:
+ return TypeIds.enumById;
+ case 1:
+ return TypeIds.namedEnum;
+ case 2:
+ return TypeIds.ext;
+ case 3:
+ return TypeIds.namedExt;
+ case 4:
+ return TypeIds.typedUnion;
+ case 5:
+ return TypeIds.namedUnion;
+ default:
+ throw StateError('Unsupported TypeDef kind code $kindCode.');
+ }
+}
+
final class TypeInfo {
final Type type;
final RegistrationKind kind;
@@ -588,25 +639,47 @@ final class TypeResolver {
required LinkedHashMap<TypeDef, int> typeDefIds,
required MetaStringWriter metaStringWriter,
}) {
- _wireTypeMetaEncoder.write(
- buffer,
- wireTypeMetaForResolved(resolved),
- writeTypeDef: (wireTypeMeta) => _writeTypeDef(
- buffer,
- typeDef ?? wireTypeMeta.resolvedType.typeDef!,
- typeDefIds: typeDefIds,
- ),
- writePackageMetaString: (value) => metaStringWriter.writeMetaString(
- buffer,
- value,
- ),
- writeTypeNameMetaString: (value) => metaStringWriter.writeMetaString(
- buffer,
- value,
- ),
- );
+ final wireTypeId = _wireTypeMetaEncoder.wireTypeIdFor(config, resolved);
+ buffer.writeVarUint32Small7(wireTypeId);
+ if (_wireTypeWritesUserTypeId(wireTypeId)) {
+ buffer.writeVarUint32(resolved.userTypeId!);
+ return;
+ }
+ if (_wireTypeWritesTypeDef(wireTypeId)) {
+ _writeTypeDef(buffer, typeDef ?? resolved.typeDef!,
+ typeDefIds: typeDefIds);
+ return;
+ }
+ if (_wireTypeWritesNamedType(wireTypeId)) {
+ metaStringWriter.writeMetaString(buffer, resolved.encodedNamespace!);
+ metaStringWriter.writeMetaString(buffer, resolved.encodedTypeName!);
+ }
}
+ @pragma('vm:prefer-inline')
+ bool _wireTypeWritesUserTypeId(int wireTypeId) =>
+ wireTypeId == TypeIds.enumById ||
+ wireTypeId == TypeIds.struct ||
+ wireTypeId == TypeIds.ext ||
+ wireTypeId == TypeIds.typedUnion;
+
+ @pragma('vm:prefer-inline')
+ bool _wireTypeWritesTypeDef(int wireTypeId) =>
+ wireTypeId == TypeIds.compatibleStruct ||
+ wireTypeId == TypeIds.namedCompatibleStruct ||
+ (config.compatible &&
+ (wireTypeId == TypeIds.namedEnum ||
+ wireTypeId == TypeIds.namedStruct ||
+ wireTypeId == TypeIds.namedExt ||
+ wireTypeId == TypeIds.namedUnion));
+
+ @pragma('vm:prefer-inline')
+ bool _wireTypeWritesNamedType(int wireTypeId) =>
+ wireTypeId == TypeIds.namedEnum ||
+ wireTypeId == TypeIds.namedStruct ||
+ wireTypeId == TypeIds.namedExt ||
+ wireTypeId == TypeIds.namedUnion;
+
@pragma('vm:prefer-inline')
TypeInfo readTypeMeta(
Buffer buffer, {
@@ -671,6 +744,7 @@ final class TypeResolver {
}) {
final encoded = _encodeTypeDef(
kind: kind,
+ evolving: evolving,
userTypeId: userTypeId,
encodedNamespace: encodedNamespace,
encodedTypeName: encodedTypeName,
@@ -687,24 +761,44 @@ final class TypeResolver {
Uint8List _encodeTypeDef({
required RegistrationKind kind,
+ required bool evolving,
required int? userTypeId,
required EncodedMetaString? encodedNamespace,
required EncodedMetaString? encodedTypeName,
required List<FieldInfo> fields,
}) {
final metaBuffer = Buffer();
- var classHeader = fields.length;
+ final byName = userTypeId == null &&
+ encodedNamespace != null &&
+ encodedTypeName != null;
+ final typeId = _typeDefTypeId(kind, byName: byName, evolving: evolving);
+ if (!_isStructTypeDefKind(typeId) && fields.isNotEmpty) {
+ throw StateError(
+ 'Non-struct TypeDef $typeId cannot carry field metadata.');
+ }
+ var classHeader = 0;
metaBuffer.writeByte(0xff);
- if (fields.length >= typeDefSmallFieldCountThreshold) {
- classHeader = typeDefSmallFieldCountThreshold;
- metaBuffer.writeVarUint32Small7(
- fields.length - typeDefSmallFieldCountThreshold,
- );
+ if (_isStructTypeDefKind(typeId)) {
+ final inlineFieldCount = fields.length >= typeDefSmallFieldCountThreshold
+ ? typeDefSmallFieldCountThreshold
+ : fields.length;
+ classHeader = typeDefStructFlag | inlineFieldCount;
+ if (typeId == TypeIds.compatibleStruct ||
+ typeId == TypeIds.namedCompatibleStruct) {
+ classHeader |= typeDefCompatibleFlag;
+ }
+ if (fields.length >= typeDefSmallFieldCountThreshold) {
+ metaBuffer.writeVarUint32Small7(
+ fields.length - typeDefSmallFieldCountThreshold,
+ );
+ }
+ if (byName) {
+ classHeader |= typeDefRegisterByNameFlag;
+ }
+ } else {
+ classHeader = _nonStructTypeDefKindCode(typeId);
}
- if (userTypeId == null &&
- encodedNamespace != null &&
- encodedTypeName != null) {
- classHeader |= typeDefRegisterByNameFlag;
+ if (byName) {
_writeTypeDefName(
metaBuffer,
encodedNamespace.bytes,
@@ -720,17 +814,18 @@ final class TypeResolver {
),
);
} else {
- metaBuffer.writeUint8(_typeDefTypeId(kind));
metaBuffer.writeVarUint32(userTypeId!);
}
metaBuffer.toBytes()[0] = classHeader;
- for (final field in fields) {
- _writeTypeDefField(metaBuffer, field);
+ if (_isStructTypeDefKind(typeId)) {
+ for (final field in fields) {
+ _writeTypeDefField(metaBuffer, field);
+ }
}
final body = metaBuffer.toBytes();
final buffer = Buffer();
buffer.writeInt64(
- typeDefHeader(body, hasFieldsMeta: fields.isNotEmpty),
+ typeDefHeader(body),
);
if (body.length >= 0xff) {
buffer.writeVarUint32(body.length - 0xff);
@@ -805,16 +900,23 @@ final class TypeResolver {
target.writeBytes(bytes);
}
- int _typeDefTypeId(RegistrationKind kind) {
+ int _typeDefTypeId(
+ RegistrationKind kind, {
+ required bool byName,
+ required bool evolving,
+ }) {
switch (kind) {
case RegistrationKind.struct:
- return TypeIds.struct;
+ if (byName) {
+ return evolving ? TypeIds.namedCompatibleStruct : TypeIds.namedStruct;
+ }
+ return evolving ? TypeIds.compatibleStruct : TypeIds.struct;
case RegistrationKind.enumType:
- return TypeIds.enumById;
+ return byName ? TypeIds.namedEnum : TypeIds.enumById;
case RegistrationKind.ext:
- return TypeIds.ext;
+ return byName ? TypeIds.namedExt : TypeIds.ext;
case RegistrationKind.union:
- return TypeIds.typedUnion;
+ return byName ? TypeIds.namedUnion : TypeIds.typedUnion;
case RegistrationKind.builtin:
throw StateError('Built-in types do not write TypeDef metadata.');
}
@@ -849,12 +951,16 @@ final class TypeResolver {
final header = TypeHeader(buffer.readInt64());
final expectedTypeDef = expectedType?.typeDef;
if (expectedTypeDef != null && expectedTypeDef.header == header.value) {
+ // Header-cache hits intentionally skip without rehashing. Entries reach this cache only
+ // after a successful TypeDef parse and 52-bit body-hash validation.
header.skipRemaining(buffer);
sharedTypes.add(expectedType!);
return wireTypeMetaForResolved(expectedType);
}
final cached = _parsedTypeMetaCache.lookup(header);
if (cached != null) {
+ // Header-cache hits intentionally skip without rehashing. Entries reach this cache only
+ // after a successful TypeDef parse and 52-bit body-hash validation.
header.skipRemaining(buffer);
sharedTypes.add(cached);
return wireTypeMetaForResolved(cached);
@@ -866,33 +972,69 @@ final class TypeResolver {
}
TypeInfo _readTypeDefWithHeader(Buffer buffer, TypeHeader header) {
+ header.validateGlobal();
final metaSize = header.readMetaSize(buffer);
- final metaBytes = Buffer.wrap(buffer.readBytes(metaSize));
+ final metaBody = buffer.readBytes(metaSize);
+ final metaBytes = Buffer.wrap(metaBody);
final classHeader = metaBytes.readUint8();
- var fieldCount = classHeader & typeDefSmallFieldCountThreshold;
- if (fieldCount == typeDefSmallFieldCountThreshold) {
- fieldCount += metaBytes.readVarUint32Small7();
+ final isStruct = (classHeader & typeDefStructFlag) != 0;
+ var fieldCount = 0;
+ bool byName;
+ int typeId;
+ if (isStruct) {
+ byName = (classHeader & typeDefRegisterByNameFlag) != 0;
+ final compatible = (classHeader & typeDefCompatibleFlag) != 0;
+ if (byName) {
+ typeId =
+ compatible ? TypeIds.namedCompatibleStruct : TypeIds.namedStruct;
+ } else {
+ typeId = compatible ? TypeIds.compatibleStruct : TypeIds.struct;
+ }
+ fieldCount = classHeader & typeDefSmallFieldCountThreshold;
+ if (fieldCount == typeDefSmallFieldCountThreshold) {
+ fieldCount += metaBytes.readVarUint32Small7();
+ }
+ } else {
+ if ((classHeader & 0x70) != 0) {
+ throw StateError('Invalid TypeDef kind header.');
+ }
+ typeId = _nonStructTypeDefTypeId(classHeader & 0x0f);
+ byName = _isNamedTypeDefKind(typeId);
}
- final byName = (classHeader & typeDefRegisterByNameFlag) != 0;
final encodedNamespace =
byName ? _readTypeDefName(metaBytes, packageNameEncoding) : null;
final encodedTypeName =
byName ? _readTypeDefName(metaBytes, typeNameEncoding) : null;
int? userTypeId;
if (!byName) {
- metaBytes.readUint8();
userTypeId = metaBytes.readVarUint32();
}
final fields = <FieldInfo>[];
for (var i = 0; i < fieldCount; i += 1) {
fields.add(_readTypeDefField(metaBytes));
}
+ if (!isStruct && fields.isNotEmpty) {
+ throw StateError('Non-struct TypeDef cannot carry field metadata.');
+ }
+ if (metaBytes.readableBytes != 0) {
+ throw StateError('Invalid TypeDef metadata size.');
+ }
+ header.validateBodyHash(metaBody);
final resolved = userTypeId != null
? resolveUserById(userTypeId)
: resolveUserByEncodedName(
encodedNamespace!,
encodedTypeName!,
);
+ if (resolved.typeDef?.header != header.value &&
+ _typeDefTypeId(
+ resolved.kind,
+ byName: byName,
+ evolving: resolved.typeDef?.evolving ?? false,
+ ) !=
+ typeId) {
+ throw StateError('TypeDef kind does not match registered type metadata.');
+ }
if (resolved.kind != RegistrationKind.struct) {
return resolved;
}
diff --git a/dart/packages/fory/lib/src/util/hash_util.dart b/dart/packages/fory/lib/src/util/hash_util.dart
index c08372b27..834f6d620 100644
--- a/dart/packages/fory/lib/src/util/hash_util.dart
+++ b/dart/packages/fory/lib/src/util/hash_util.dart
@@ -25,10 +25,9 @@ import 'package:fory/src/meta/type_ids.dart';
import 'package:fory/src/types/int64.dart';
import 'package:fory/src/types/uint64.dart';
-const int _typeDefCompressMetaFlag = 1 << 9;
-const int _typeDefHasFieldsMetaFlag = 1 << 8;
+const int _typeDefCompressMetaFlag = 1 << 8;
const int _typeDefMetaSizeMask = 0xff;
-const int _typeDefHashShift = 14;
+const int _typeDefHashShift = 12;
final Uint64 _metaStringHashMask = Uint64.fromWords(0xffffff00, 0xffffffff);
final Uint64 _c1 = Uint64.fromWords(0x114253d5, 0x87c37b91);
@@ -154,7 +153,6 @@ Int64 metaStringHash(List<int> bytes, {int encoding = 0}) {
Int64 typeDefHeader(
List<int> bytes, {
- required bool hasFieldsMeta,
bool compressed = false,
}) {
final hash = _int64FromUint64(
@@ -164,9 +162,6 @@ Int64 typeDefHeader(
if (compressed) {
header = header | _typeDefCompressMetaFlag;
}
- if (hasFieldsMeta) {
- header = header | _typeDefHasFieldsMetaFlag;
- }
header = header |
(bytes.length > _typeDefMetaSizeMask
? _typeDefMetaSizeMask
diff --git a/dart/packages/fory/test/decimal_serializer_test.dart b/dart/packages/fory/test/decimal_serializer_test.dart
index ed009b5ad..6c42175aa 100644
--- a/dart/packages/fory/test/decimal_serializer_test.dart
+++ b/dart/packages/fory/test/decimal_serializer_test.dart
@@ -88,14 +88,14 @@ void main() {
test('rejects non-canonical big decimal payloads', () {
final fory = Fory();
final zeroBigEncoding = Uint8List.fromList(<int>[
- 0x02,
+ 0x01,
0xff,
TypeIds.decimal,
0x00,
0x01,
]);
final trailingZeroPayload = Uint8List.fromList(<int>[
- 0x02,
+ 0x01,
0xff,
TypeIds.decimal,
0x00,
diff --git a/dart/packages/fory/test/time_serializer_test.dart b/dart/packages/fory/test/time_serializer_test.dart
index 8715f845d..0da5524e2 100644
--- a/dart/packages/fory/test/time_serializer_test.dart
+++ b/dart/packages/fory/test/time_serializer_test.dart
@@ -98,7 +98,7 @@ void main() {
final bytes = fory.serialize(value);
expect(
- bytes, equals(Uint8List.fromList([0x02, 0xff, TypeIds.date, 0x01])));
+ bytes, equals(Uint8List.fromList([0x01, 0xff, TypeIds.date, 0x01])));
expect(fory.deserialize<LocalDate>(bytes), equals(value));
});
diff --git a/dart/packages/fory/test/xlang_protocol_test.dart b/dart/packages/fory/test/xlang_protocol_test.dart
index be3b9bb76..0f182af5e 100644
--- a/dart/packages/fory/test/xlang_protocol_test.dart
+++ b/dart/packages/fory/test/xlang_protocol_test.dart
@@ -20,13 +20,33 @@
import 'dart:typed_data';
import 'package:fory/fory.dart';
+import 'package:fory/src/context/read_context.dart';
+import 'package:fory/src/context/write_context.dart';
+import 'package:fory/src/meta/type_meta.dart';
+import 'package:fory/src/resolver/type_resolver.dart';
+import 'package:fory/src/serializer/serializer.dart';
+import 'package:fory/src/types/int64.dart';
+import 'package:fory/src/util/hash_util.dart';
import 'package:test/test.dart';
+final class _CacheTestSerializer extends Serializer<Object?> {
+ const _CacheTestSerializer();
+
+ @override
+ bool get supportsRef => false;
+
+ @override
+ Object? read(ReadContext context) => null;
+
+ @override
+ void write(WriteContext context, Object? value) {}
+}
+
void main() {
group('xlang protocol regressions', () {
test('deserializes NONE wire values as null', () {
final fory = Fory();
- final bytes = Uint8List.fromList(<int>[0x02, 0xff, TypeIds.none]);
+ final bytes = Uint8List.fromList(<int>[0x01, 0xff, TypeIds.none]);
expect(fory.deserialize<Object?>(bytes), isNull);
expect(fory.deserialize<Null>(bytes), isNull);
@@ -77,7 +97,7 @@ void main() {
final fory = Fory();
final bytes = fory.serializeBuiltin(7, wireTypeId: TypeIds.varInt32);
- expect(bytes[0], equals(0x02));
+ expect(bytes[0], equals(0x01));
expect(bytes[1], equals(0xff));
expect(bytes[2], equals(TypeIds.varInt32));
expect(fory.deserialize<int>(bytes), equals(7));
@@ -86,7 +106,7 @@ void main() {
test('rejects out-of-band xlang payload headers', () {
final fory = Fory();
final bytes = Uint8List.fromList(fory.serialize('value'));
- bytes[0] |= 0x04;
+ bytes[0] |= 0x02;
expect(
() => fory.deserialize<String>(bytes),
@@ -99,5 +119,57 @@ void main() {
),
);
});
+
+ test('parsed TypeDef cache stops publishing at capacity', () {
+ const resolved = TypeInfo(
+ type: Object,
+ kind: RegistrationKind.builtin,
+ typeId: TypeIds.struct,
+ supportsRef: false,
+ serializer: _CacheTestSerializer(),
+ structSerializer: null,
+ userTypeId: null,
+ namespace: null,
+ typeName: null,
+ encodedNamespace: null,
+ encodedTypeName: null,
+ typeDef: null,
+ remoteTypeDef: null,
+ );
+ final cache = ParsedTypeMetaCache();
+ for (var i = 0; i < ParsedTypeMetaCache.maxEntries; i++) {
+ cache.remember(TypeHeader(Int64(i)), resolved);
+ }
+
+ expect(
+ cache.lookup(TypeHeader(Int64(ParsedTypeMetaCache.maxEntries - 1))),
+ same(resolved),
+ );
+ final uncached = TypeHeader(Int64(ParsedTypeMetaCache.maxEntries));
+ cache.remember(uncached, resolved);
+
+ expect(cache.lookup(uncached), isNull);
+ });
+
+ test('validates parsed TypeDef body hash before caching', () {
+ final body = Uint8List.fromList(<int>[0x80]);
+ final header = TypeHeader(typeDefHeader(body));
+ final valid = Buffer.wrap(body);
+ header.skipRemaining(valid);
+ expect(valid.readableBytes, equals(0));
+
+ final malformed = Uint8List.fromList(body);
+ malformed[0] ^= 1;
+ expect(
+ () => header.validateBodyHash(malformed),
+ throwsA(
+ isA<StateError>().having(
+ (error) => error.toString(),
+ 'message',
+ contains('metadata hash'),
+ ),
+ ),
+ );
+ });
});
}
diff --git a/docs/specification/java_serialization_spec.md b/docs/specification/java_serialization_spec.md
index 2bac7bc0d..c52695f1c 100644
--- a/docs/specification/java_serialization_spec.md
+++ b/docs/specification/java_serialization_spec.md
@@ -44,14 +44,14 @@ Java native serialization writes a one byte bitmap header. The header layout mir
bitmap and uses the same flag bits.
```
-| 5 bits | 1 bit | 1 bit | 1 bit |
-+--------------+-------+-------+-------+
-| reserved | oob | xlang | null |
+| 6 bits | 1 bit | 1 bit |
++---------------+-------+-------+
+| reserved | oob | xlang |
```
-- null flag: 1 when object is null, 0 otherwise. If object is null, other bits are not set.
-- xlang flag: 1 when serialization uses xlang format, 0 when serialization uses Java native format.
-- oob flag: 1 when `BufferCallback` is not null, 0 otherwise.
+- xlang flag: bit 0, set when serialization uses xlang format and clear for Java native format.
+- oob flag: bit 1, set when `BufferCallback` is not null.
+- reserved bits: bits 2-7, must be zero.
The header is always a single byte; no language ID is written.
@@ -202,32 +202,37 @@ when shared meta is enabled, or referenced by index when already seen.
Header layout (lower bits on the right):
```
-| 50-bit hash | 4 bits reserved | 1 bit compress | 1 bit has_fields_meta | 8-bit size |
+| 52-bit hash | 3 bits reserved | 1 bit compress | 8-bit size |
```
- size: lower 8 bits. If size equals the mask (0xFF), write extra size as varuint32 and add it.
-- compress: set when payload is compressed.
-- has_fields_meta: set when field metadata is present.
-- reserved: bits 10-13 are reserved for future use and must be zero.
-- hash: 50-bit hash of the payload and flags.
+- compress: bit 8, set when payload is compressed.
+- reserved: bits 9-11 are reserved for future use and must be zero.
+- hash: 52-bit hash of the payload.
### Class meta bytes
Class meta encodes a linearized class hierarchy (from parent to leaf) and field metadata:
```
-| num_classes | class_layer_0 | class_layer_1 | ... |
+| root_kind_and_num_classes | class_layer_0 | class_layer_1 | ... |
class_layer:
| num_fields << 1 | registered_flag | [type_id if registered] |
| namespace | type_name | field_infos |
```
-- `num_classes` stores `(num_layers - 1)` in a single byte.
- - If it equals `0b1111`, read an extra varuint32 small7 and add it.
+- `root_kind_and_num_classes` stores the root TypeDef kind in the high four bits and
+ `(num_layers - 1)` in the low four bits.
+ - Root kind codes are `STRUCT=0`, `COMPATIBLE_STRUCT=1`, `NAMED_STRUCT=2`,
+ `NAMED_COMPATIBLE_STRUCT=3`, `ENUM=4`, `NAMED_ENUM=5`, `EXT=6`, `NAMED_EXT=7`,
+ `TYPED_UNION=8`, and `NAMED_UNION=9`.
+ - Kind codes `10-14` are reserved and `15` is an extended-kind escape rejected until defined.
+ - If the low four bits equal `0b1111`, read an extra varuint32 small7 and add it.
- The actual number of layers is `num_classes + 1`.
- `registered_flag` is 1 if the class is registered by numeric ID.
-- If registered by ID, the class type ID follows (varuint32 small7).
+- If registered by ID, the one-byte class type ID follows. For user-registered ID kinds, the
+ user type ID follows as varuint32.
- If registered by name or unregistered, namespace and type name are written as meta strings.
### Field info
diff --git a/docs/specification/xlang_serialization_spec.md b/docs/specification/xlang_serialization_spec.md
index 24adbe333..b1743918a 100644
--- a/docs/specification/xlang_serialization_spec.md
+++ b/docs/specification/xlang_serialization_spec.md
@@ -323,15 +323,14 @@ Detailed byte layout:
```
Byte 0: Bitmap flags
- - Bit 0: null flag (0x01)
- - Bit 1: xlang flag (0x02)
- - Bit 2: oob flag (0x04)
- - Bits 3-7: reserved
+ - Bit 0: xlang flag (0x01)
+ - Bit 1: oob flag (0x02)
+ - Bits 2-7: reserved
```
-- **null flag** (bit 0): 1 when object is null, 0 otherwise. If an object is null, only this flag is set.
-- **xlang flag** (bit 1): 1 when serialization uses Fory xlang format, 0 when serialization uses Fory language-native format.
-- **oob flag** (bit 2): 1 when out-of-band serialization is enabled (BufferCallback is not null), 0 otherwise.
+- **xlang flag** (bit 0): 1 when serialization uses Fory xlang format, 0 when serialization uses Fory language-native format.
+- **oob flag** (bit 1): 1 when out-of-band serialization is enabled (BufferCallback is not null), 0 otherwise.
+- **reserved bits** (bits 2-7): must be zero.
All data is encoded in little-endian format.
@@ -536,12 +535,11 @@ The 8-byte header is a little-endian uint64:
- Low 8 bits: meta size (number of bytes in the TypeDef body).
- If meta size >= 0xFF, the low 8 bits are set to 0xFF and an extra
`varuint32(meta_size - 0xFF)` follows immediately after the header.
-- Bit 8: `HAS_FIELDS_META` (1 = fields metadata present).
-- Bit 9: `COMPRESS_META` is reserved for a future xlang metadata-compression extension.
+- Bit 8: `COMPRESS_META` is reserved for a future xlang metadata-compression extension.
Current xlang writers MUST leave this bit unset and current xlang readers MUST treat a set bit
as unsupported.
-- Bits 10-13: reserved for future extension (must be zero).
-- High 50 bits: hash of the TypeDef body.
+- Bits 9-11: reserved for future extension (must be zero).
+- High 52 bits: hash of the TypeDef body.
#### TypeDef body
@@ -551,12 +549,30 @@ TypeDef body has a single layer (fields are flattened in class hierarchy order):
| meta header (1 byte) | type spec | field info ... |
```
-Meta header byte:
+Meta header byte for struct TypeDefs:
+- Bit 7: `IS_STRUCT` (1).
+- Bit 6: `COMPATIBLE`.
+- Bit 5: `REGISTER_BY_NAME` (1 = namespace + type name, 0 = numeric user type ID).
- Bits 0-4: `num_fields` (0-30).
- If `num_fields == 31`, read an extra `varuint32` and add it.
-- Bit 5: `REGISTER_BY_NAME` (1 = namespace + type name, 0 = numeric type ID).
-- Bits 6-7: reserved.
+
+Meta header byte for non-struct TypeDefs:
+
+- Bit 7: `IS_STRUCT` (0).
+- Bits 4-6: reserved (must be zero).
+- Bits 0-3: kind code.
+
+Non-struct kind codes:
+
+- `0`: `ENUM`
+- `1`: `NAMED_ENUM`
+- `2`: `EXT`
+- `3`: `NAMED_EXT`
+- `4`: `TYPED_UNION`
+- `5`: `NAMED_UNION`
+- `6-14`: reserved
+- `15`: extended-kind escape, rejected until defined
Type spec:
@@ -564,7 +580,7 @@ Type spec:
- `namespace` meta string
- `type_name` meta string
- Otherwise:
- - `type_id` as `varuint32` (small7)
+ - user type ID as `varuint32`
Field info list:
diff --git a/go/fory/buffer.go b/go/fory/buffer.go
index c51071fd1..2e0f13655 100644
--- a/go/fory/buffer.go
+++ b/go/fory/buffer.go
@@ -1224,8 +1224,8 @@ func (b *ByteBuffer) ReadVarint32(err *Error) int32 {
// UnsafeReadVarint32 reads a varint32 without bounds checking.
// Caller must ensure remaining() >= 5 before calling.
-func (b *ByteBuffer) UnsafeReadVarint32() int32 {
- u := b.readVarUint32Fast()
+func (b *ByteBuffer) UnsafeReadVarint32(err *Error) int32 {
+ u := b.readVarUint32Fast(err)
v := int32(u >> 1)
if u&1 != 0 {
v = ^v
@@ -1246,8 +1246,8 @@ func (b *ByteBuffer) UnsafeReadVarint64() int64 {
// UnsafeReadVarUint32 reads a VarUint32 without bounds checking.
// Caller must ensure remaining() >= 5 before calling.
-func (b *ByteBuffer) UnsafeReadVarUint32() uint32 {
- return b.readVarUint32Fast()
+func (b *ByteBuffer) UnsafeReadVarUint32(err *Error) uint32 {
+ return b.readVarUint32Fast(err)
}
// UnsafeReadVarUint64 reads a VarUint64 without bounds checking.
@@ -1259,13 +1259,13 @@ func (b *ByteBuffer) UnsafeReadVarUint64() uint64 {
// ReadVarUint32 reads a VarUint32 and sets error on bounds violation
func (b *ByteBuffer) ReadVarUint32(err *Error) uint32 {
if b.remaining() >= 8 { // Need 8 bytes for bulk uint64 read in fast path
- return b.readVarUint32Fast()
+ return b.readVarUint32Fast(err)
}
return b.readVarUint32Slow(err)
}
// Fast path reading (when the remaining bytes are sufficient)
-func (b *ByteBuffer) readVarUint32Fast() uint32 {
+func (b *ByteBuffer) readVarUint32Fast(err *Error) uint32 {
// Single instruction load using unsafe pointer cast (little-endian only)
// On big-endian systems, use binary.LittleEndian which the compiler optimizes
var bulk uint64
@@ -1288,6 +1288,13 @@ func (b *ByteBuffer) readVarUint32Fast() uint32 {
result |= uint32((bulk >> 3) & 0xFE00000)
readLength = 4
if (bulk & 0x80000000) != 0 {
+ fifth := byte(bulk >> 32)
+ if fifth > 0x0F {
+ if err != nil {
+ *err = DeserializationError("VarUint32 overflow")
+ }
+ return 0
+ }
result |= uint32((bulk >> 4) & 0xF0000000)
readLength = 5
}
@@ -1310,6 +1317,12 @@ func (b *ByteBuffer) readVarUint32Slow(err *Error) uint32 {
}
byteVal := b.data[b.readerIndex]
b.readerIndex++
+ if shift == 28 && byteVal > 0x0F {
+ if err != nil {
+ *err = DeserializationError("VarUint32 overflow")
+ }
+ return 0
+ }
result |= (uint32(byteVal) & 0x7F) << shift
if byteVal < 0x80 {
break
@@ -1475,16 +1488,16 @@ func (b *ByteBuffer) readVarUint32Small14(err *Error) uint32 {
readIdx++
value |= (four >> 1) & 0x3f80
if four&0x8000 != 0 {
- return b.continueReadVarUint32(readIdx, four, value)
+ return b.continueReadVarUint32(readIdx, four, value, err)
}
}
b.readerIndex = readIdx
return value
}
- return uint32(b.readVaruint36Slow(err))
+ return b.readVarUint32Slow(err)
}
-func (b *ByteBuffer) continueReadVarUint32(readIdx int, bulkRead, value uint32) uint32 {
+func (b *ByteBuffer) continueReadVarUint32(readIdx int, bulkRead, value uint32, err *Error) uint32 {
readIdx++
value |= (bulkRead >> 2) & 0x1fc000
if bulkRead&0x800000 != 0 {
@@ -1492,6 +1505,12 @@ func (b *ByteBuffer) continueReadVarUint32(readIdx int, bulkRead, value uint32)
value |= (bulkRead >> 3) & 0xfe00000
if bulkRead&0x80000000 != 0 {
v := b.data[readIdx]
+ if v > 0x0F {
+ if err != nil {
+ *err = DeserializationError("VarUint32 overflow")
+ }
+ return 0
+ }
readIdx++
value |= uint32(v&0x7F) << 28
}
diff --git a/go/fory/buffer_test.go b/go/fory/buffer_test.go
index a65d49a7a..b4f202238 100644
--- a/go/fory/buffer_test.go
+++ b/go/fory/buffer_test.go
@@ -18,6 +18,7 @@
package fory
import (
+ "bytes"
"testing"
"github.com/stretchr/testify/require"
@@ -111,3 +112,29 @@ func TestUnsafePutVarUint32PhysicalWriteWidth(t *testing.T) {
"byte at index %d is outside the 8-byte reserved window and must not be written", i)
}
}
+
+func TestReadVarUint32RejectsOverflowFifthByte(t *testing.T) {
+ for _, data := range [][]byte{
+ {0x80, 0x80, 0x80, 0x80, 0x10},
+ {0x80, 0x80, 0x80, 0x80, 0x10, 0, 0, 0},
+ } {
+ buf := NewByteBuffer(data)
+ var err Error
+ _ = buf.ReadVarUint32(&err)
+ require.True(t, err.HasError(), "expected overflow error for %v", data)
+ }
+}
+
+func TestReadVarUint32Small7RejectsOverflowFifthByte(t *testing.T) {
+ buf := NewByteBuffer([]byte{0x80, 0x80, 0x80, 0x80, 0x10})
+ var err Error
+ _ = buf.ReadVarUint32Small7(&err)
+ require.True(t, err.HasError())
+}
+
+func TestReadVarUint32Small7StreamRejectsOverflowFifthByte(t *testing.T) {
+ buf := NewByteBufferFromReader(bytes.NewReader([]byte{0x80, 0x80, 0x80, 0x80, 0x10}), 4)
+ var err Error
+ _ = buf.ReadVarUint32Small7(&err)
+ require.True(t, err.HasError())
+}
diff --git a/go/fory/decimal.go b/go/fory/decimal.go
index 92d6e15be..5bc3a0d54 100644
--- a/go/fory/decimal.go
+++ b/go/fory/decimal.go
@@ -77,8 +77,8 @@ func (s decimalSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(DECIMAL) {
+ return
}
if ctx.HasError() {
return
diff --git a/go/fory/enum.go b/go/fory/enum.go
index 1a1e63c61..b8f615b97 100644
--- a/go/fory/enum.go
+++ b/go/fory/enum.go
@@ -85,7 +85,27 @@ func (s *enumSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
}
}
if readType {
- _ = ctx.buffer.ReadUint8(err)
+ typeID := uint32(ctx.buffer.ReadUint8(err))
+ if ctx.HasError() {
+ return
+ }
+ internalID := TypeId(typeID)
+ if internalID != ENUM && internalID != NAMED_ENUM {
+ ctx.SetError(TypeMismatchError(internalID, ENUM))
+ return
+ }
+ typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err)
+ if ctx.HasError() {
+ return
+ }
+ if typeInfo == nil || typeInfo.Type != s.type_ {
+ var actualType reflect.Type
+ if typeInfo != nil {
+ actualType = typeInfo.Type
+ }
+ ctx.SetError(DeserializationErrorf("enum type mismatch: expected %v, got %v", s.type_, actualType))
+ return
+ }
}
if ctx.HasError() {
return
diff --git a/go/fory/enum_test.go b/go/fory/enum_test.go
new file mode 100644
index 000000000..f92c4d086
--- /dev/null
+++ b/go/fory/enum_test.go
@@ -0,0 +1,98 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fory
+
+import (
+ "reflect"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+type auditEnum int32
+type otherAuditEnum int32
+type namedAuditEnum int32
+
+func TestEnumReadConsumesRegisteredTypeInfo(t *testing.T) {
+ f := NewFory(WithXlang(true))
+ require.NoError(t, f.RegisterEnum(auditEnum(0), 101))
+ enumType := reflect.TypeOf(auditEnum(0))
+ serializer, err := f.typeResolver.getSerializerByType(enumType, false)
+ require.NoError(t, err)
+ typeInfo := f.typeResolver.typesInfo[enumType]
+ require.NotNil(t, typeInfo)
+
+ buf := NewByteBuffer(nil)
+ bufErr := &Error{}
+ f.typeResolver.WriteTypeInfo(buf, typeInfo, bufErr)
+ require.NoError(t, bufErr.CheckError())
+ buf.WriteVarUint32Small7(2)
+
+ f.readCtx.SetData(buf.Bytes())
+ var result auditEnum
+ serializer.Read(f.readCtx, RefModeNone, true, false, reflect.ValueOf(&result).Elem())
+ require.NoError(t, f.readCtx.CheckError())
+ require.Equal(t, auditEnum(2), result)
+ require.Equal(t, buf.WriterIndex(), f.readCtx.Buffer().ReaderIndex())
+}
+
+func TestEnumReadRejectsMismatchedRegisteredTypeInfo(t *testing.T) {
+ f := NewFory(WithXlang(true))
+ require.NoError(t, f.RegisterEnum(auditEnum(0), 101))
+ require.NoError(t, f.RegisterEnum(otherAuditEnum(0), 102))
+ enumType := reflect.TypeOf(auditEnum(0))
+ otherType := reflect.TypeOf(otherAuditEnum(0))
+ serializer, err := f.typeResolver.getSerializerByType(enumType, false)
+ require.NoError(t, err)
+ otherTypeInfo := f.typeResolver.typesInfo[otherType]
+ require.NotNil(t, otherTypeInfo)
+
+ buf := NewByteBuffer(nil)
+ bufErr := &Error{}
+ f.typeResolver.WriteTypeInfo(buf, otherTypeInfo, bufErr)
+ require.NoError(t, bufErr.CheckError())
+ buf.WriteVarUint32Small7(2)
+
+ f.readCtx.SetData(buf.Bytes())
+ var result auditEnum
+ serializer.Read(f.readCtx, RefModeNone, true, false, reflect.ValueOf(&result).Elem())
+ require.Error(t, f.readCtx.CheckError())
+}
+
+func TestNamedEnumReadConsumesNamedTypeInfo(t *testing.T) {
+ f := NewFory(WithXlang(true))
+ require.NoError(t, f.RegisterNamedEnum(namedAuditEnum(0), "example.NamedAuditEnum"))
+ enumType := reflect.TypeOf(namedAuditEnum(0))
+ serializer, err := f.typeResolver.getSerializerByType(enumType, false)
+ require.NoError(t, err)
+ typeInfo := f.typeResolver.typesInfo[enumType]
+ require.NotNil(t, typeInfo)
+
+ buf := NewByteBuffer(nil)
+ bufErr := &Error{}
+ f.typeResolver.WriteTypeInfo(buf, typeInfo, bufErr)
+ require.NoError(t, bufErr.CheckError())
+ buf.WriteVarUint32Small7(3)
+
+ f.readCtx.SetData(buf.Bytes())
+ var result namedAuditEnum
+ serializer.Read(f.readCtx, RefModeNone, true, false, reflect.ValueOf(&result).Elem())
+ require.NoError(t, f.readCtx.CheckError())
+ require.Equal(t, namedAuditEnum(3), result)
+ require.Equal(t, buf.WriterIndex(), f.readCtx.Buffer().ReaderIndex())
+}
diff --git a/go/fory/fory.go b/go/fory/fory.go
index 61723d190..f50fe1f77 100644
--- a/go/fory/fory.go
+++ b/go/fory/fory.go
@@ -39,9 +39,9 @@ var ErrNoSerializer = errors.New("fory: no serializer registered for type")
// Bitmap flags for protocol header
const (
- IsNilFlag = 1 << 0
- XLangFlag = 1 << 1
- OutOfBandFlag = 1 << 2
+ XLangFlag = 1 << 0
+ OutOfBandFlag = 1 << 1
+ headerFlagMask = XLangFlag | OutOfBandFlag
)
// ============================================================================
@@ -185,6 +185,9 @@ func New(opts ...Option) *Fory {
f.readCtx.refResolver = f.refResolver
f.readCtx.compatible = f.config.Compatible
f.readCtx.xlang = f.config.IsXlang
+ if f.config.IsXlang {
+ f.readCtx.rootHeader = XLangFlag
+ }
return f
}
@@ -487,13 +490,6 @@ func (f *Fory) Reset() {
// For thread-safe usage, use threadsafe.Fory which copies the data internally.
func (f *Fory) Serialize(value any) ([]byte, error) {
defer f.resetWriteState()
- // Check if value is nil interface OR a nil pointer/slice/map/etc.
- // In Go, `*int32(nil)` wrapped in `any` is NOT equal to `nil`, but we need to serialize it as null.
- if isNilValue(value) {
- // Use Java-compatible null format: 3 bytes (magic + bitmap with isNilFlag)
- writeNullHeader(f.writeCtx)
- return f.writeCtx.buffer.GetByteSlice(0, f.writeCtx.buffer.writerIndex), nil
- }
// WriteData protocol header
writeHeader(f.writeCtx, f.config)
@@ -521,16 +517,11 @@ func (f *Fory) Deserialize(data []byte, v any) error {
defer f.resetReadState()
f.readCtx.SetData(data)
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
return f.readCtx.TakeError()
}
- // Check if the serialized object is null
- if isNull {
- return nil
- }
-
// Deserialize the value - TypeMeta is read inline using streaming protocol
target := reflect.ValueOf(v).Elem()
f.readCtx.ReadValue(target, RefModeTracking, true)
@@ -561,13 +552,6 @@ func (f *Fory) resetWriteState() {
// This is useful when you need to write multiple serialized values to the same buffer.
// Returns error if serialization fails.
func (f *Fory) SerializeTo(buf *ByteBuffer, value any) error {
- // Handle nil values
- if isNilValue(value) {
- // Use Java-compatible null format: 1 byte (bitmap with isNilFlag)
- buf.WriteByte_(IsNilFlag)
- return nil
- }
-
defer f.resetWriteState()
// Temporarily swap buffer
@@ -625,18 +609,12 @@ func (f *Fory) DeserializeFrom(buf *ByteBuffer, v any) error {
origBuffer := f.readCtx.buffer
f.readCtx.buffer = buf
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
f.readCtx.buffer = origBuffer
return f.readCtx.TakeError()
}
- // Check if the serialized object is null
- if isNull {
- f.readCtx.buffer = origBuffer
- return nil
- }
-
// Deserialize the value - TypeMeta is read inline using streaming protocol
target := reflect.ValueOf(v).Elem()
f.readCtx.ReadValue(target, RefModeTracking, true)
@@ -731,21 +709,11 @@ func (f *Fory) DeserializeWithCallbackBuffers(buffer *ByteBuffer, v any, buffers
}
// ReadData and validate header
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
return f.readCtx.TakeError()
}
- // Check if the serialized object is null
- if isNull {
- // v must be a pointer so we can set it to nil
- rv := reflect.ValueOf(v)
- if rv.Kind() == reflect.Ptr && !rv.IsNil() {
- rv.Elem().Set(reflect.Zero(rv.Elem().Type()))
- }
- return nil
- }
-
// v must be a pointer so we can deserialize into it
if v == nil {
return fmt.Errorf("v cannot be nil")
@@ -803,50 +771,34 @@ func writeHeader(ctx *WriteContext, config Config) {
ctx.buffer.WriteByte_(bitmap)
}
-// isNilValue checks if a value is nil, including nil pointers wrapped in any
-// In Go, `*int32(nil)` wrapped in `any` is NOT equal to `nil`, but we need to treat it as null.
-//
-//go:noinline
-func isNilValue(value any) bool {
- if value == nil {
- return true
- }
- rv := reflect.ValueOf(value)
- switch rv.Kind() {
- case reflect.Ptr, reflect.Slice, reflect.Map, reflect.Chan, reflect.Func, reflect.Interface:
- return rv.IsNil()
- }
- return false
-}
-
-// writeNullHeader writes a null object header (1 byte: bitmap with isNilFlag)
-// This is compatible with Java's null serialization format
-//
-//go:noinline
-func writeNullHeader(ctx *WriteContext) {
- ctx.buffer.WriteByte_(IsNilFlag) // bitmap with only isNilFlag set
-}
-
-// Special return value indicating null object in readHeader
-// Using math.MinInt32 to avoid conflict with -1 which is used for "no meta offset"
-const NullObjectMetaOffset int32 = -0x7FFFFFFF
-
// readHeader reads and validates the Fory protocol header
-// Returns true if the serialized object is null
// Sets error on ctx if header is invalid (use ctx.HasError() to check)
-func readHeader(ctx *ReadContext) bool {
+func readHeader(ctx *ReadContext) {
err := ctx.Err()
bitmap := ctx.buffer.ReadByte(err)
if ctx.HasError() {
- return false
+ return
}
-
- // Check if this is a null object - only bitmap with isNilFlag was written
- if (bitmap & IsNilFlag) != 0 {
- return true // is null
+ if bitmap == ctx.rootHeader {
+ return
}
+ readHeaderSlow(ctx, bitmap)
+}
- return false // not null
+//go:noinline
+func readHeaderSlow(ctx *ReadContext, bitmap byte) {
+ if bitmap&^headerFlagMask != 0 {
+ ctx.SetError(DeserializationErrorf("unsupported root header bitmap 0x%02x", bitmap))
+ return
+ }
+ if ((bitmap & XLangFlag) != 0) != ctx.xlang {
+ ctx.SetError(DeserializationErrorf("header bitmap mismatch at xlang bit"))
+ return
+ }
+ if (bitmap&OutOfBandFlag) != 0 && ctx.outOfBandBuffers == nil {
+ ctx.SetError(DeserializationErrorf("out-of-band buffers are required by root header"))
+ return
+ }
}
// ============================================================================
@@ -1025,66 +977,84 @@ func Deserialize[T any](f *Fory, data []byte, target *T) error {
f.readCtx.SetData(data)
// ReadData and validate header
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
return f.readCtx.TakeError()
}
- // Check if the serialized object is null
- if isNull {
- var zero T
- *target = zero
- return nil
- }
-
// Fast path: type switch for common types (Go compiler can optimize this)
// For primitives, read null flag, skip type ID, then read value from buffer
buf := f.readCtx.buffer
err := f.readCtx.Err()
switch t := any(target).(type) {
case *bool:
- _ = buf.ReadInt8(err) // null flag
- _ = buf.ReadUint8(err) // type ID
+ _ = buf.ReadInt8(err) // null flag
+ if !f.readCtx.readExpectedTypeID(BOOL) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadBool(err)
return f.readCtx.CheckError()
case *int8:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(INT8) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadInt8(err)
return f.readCtx.CheckError()
case *int16:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(INT16) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadInt16(err)
return f.readCtx.CheckError()
case *int32:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(VARINT32) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadVarint32(err)
return f.readCtx.CheckError()
case *int64:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(VARINT64) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadVarint64(err)
return f.readCtx.CheckError()
case *int:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if strconv.IntSize == 32 {
+ if !f.readCtx.readExpectedTypeID(VARINT32) {
+ return f.readCtx.CheckError()
+ }
+ *t = int(buf.ReadVarint32(err))
+ return f.readCtx.CheckError()
+ }
+ if !f.readCtx.readExpectedTypeID(VARINT64) {
+ return f.readCtx.CheckError()
+ }
*t = int(buf.ReadVarint64(err))
return f.readCtx.CheckError()
case *float32:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(FLOAT32) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadFloat32(err)
return f.readCtx.CheckError()
case *float64:
_ = buf.ReadInt8(err)
- _ = buf.ReadUint8(err)
+ if !f.readCtx.readExpectedTypeID(FLOAT64) {
+ return f.readCtx.CheckError()
+ }
*t = buf.ReadFloat64(err)
return f.readCtx.CheckError()
case *string:
- _ = buf.ReadInt8(err) // null flag
- _ = buf.ReadUint8(err) // type ID
+ _ = buf.ReadInt8(err) // null flag
+ if !f.readCtx.readExpectedTypeID(STRING) {
+ return f.readCtx.CheckError()
+ }
*t = f.readCtx.ReadString()
return f.readCtx.CheckError()
case *[]byte:
@@ -1116,31 +1086,31 @@ func Deserialize[T any](f *Fory, data []byte, target *T) error {
return f.readCtx.CheckError()
case *map[string]string:
*t = f.readCtx.ReadStringStringMap(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[string]int64:
*t = f.readCtx.ReadStringInt64Map(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[string]int32:
*t = f.readCtx.ReadStringInt32Map(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[string]int:
*t = f.readCtx.ReadStringIntMap(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[string]float64:
*t = f.readCtx.ReadStringFloat64Map(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[string]bool:
*t = f.readCtx.ReadStringBoolMap(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[int32]int32:
*t = f.readCtx.ReadInt32Int32Map(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[int64]int64:
*t = f.readCtx.ReadInt64Int64Map(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
case *map[int]int:
*t = f.readCtx.ReadIntIntMap(RefModeNullOnly, true)
- return nil
+ return f.readCtx.CheckError()
default:
// Slow path: use serializer-based deserialization
targetVal := reflect.ValueOf(target).Elem()
diff --git a/go/fory/fory_typed_test.go b/go/fory/fory_typed_test.go
index 810807657..127c05c89 100644
--- a/go/fory/fory_typed_test.go
+++ b/go/fory/fory_typed_test.go
@@ -132,6 +132,44 @@ func TestSerializeGenericPrimitives(t *testing.T) {
})
}
+func TestDeserializeRejectsRootTypeMismatch(t *testing.T) {
+ f := NewFory()
+
+ data := []byte{0, 0xff, byte(STRING)}
+ var result bool
+ require.Error(t, Deserialize(f, data, &result))
+
+ data = []byte{0, 0xff, byte(BOOL)}
+ var mapResult map[string]string
+ require.Error(t, Deserialize(f, data, &mapResult))
+}
+
+func TestDeserializeRejectsRootPrimitiveSliceTypeMismatch(t *testing.T) {
+ f := NewFory()
+
+ data := []byte{0, 0xff, byte(BINARY)}
+ var int32Result []int32
+ require.Error(t, Deserialize(f, data, &int32Result))
+
+ data = []byte{0, 0xff, byte(INT8_ARRAY)}
+ var byteResult []byte
+ require.Error(t, Deserialize(f, data, &byteResult))
+}
+
+func TestDeserializeByteSliceAcceptsUint8ArrayRootType(t *testing.T) {
+ f := NewFory()
+ buf := NewByteBuffer(nil)
+ buf.WriteByte(0)
+ buf.WriteInt8(NotNullValueFlag)
+ buf.WriteUint8(uint8(UINT8_ARRAY))
+ buf.WriteLength(3)
+ buf.WriteBinary([]byte{1, 2, 3})
+
+ var result []byte
+ require.NoError(t, Deserialize(f, buf.Bytes(), &result))
+ require.Equal(t, []byte{1, 2, 3}, result)
+}
+
// TestSerializeGenericComplex tests Serialize[T]/DeserializeWithCallbackBuffers[T] with complex types.
// These fall back to reflection-based serialization.
func TestSerializeGenericComplex(t *testing.T) {
diff --git a/go/fory/map.go b/go/fory/map.go
index cbce365e2..a59fd8e65 100644
--- a/go/fory/map.go
+++ b/go/fory/map.go
@@ -484,6 +484,10 @@ func (s mapSerializer) readChunk(ctx *ReadContext, mapVal reflect.Value, header
if ctx.HasError() {
return 0
}
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return 0
+ }
// Read type info if not declared
var keyTypeInfo, valueTypeInfo *TypeInfo
@@ -618,7 +622,9 @@ func readMapRefAndType(ctx *ReadContext, refMode RefMode, readType bool, value r
}
}
if readType {
- buf.ReadUint8(ctxErr)
+ if !ctx.readExpectedTypeID(MAP) {
+ return false
+ }
}
return false
}
diff --git a/go/fory/map_primitive.go b/go/fory/map_primitive.go
index 16e40f5ae..287777eae 100644
--- a/go/fory/map_primitive.go
+++ b/go/fory/map_primitive.go
@@ -81,50 +81,32 @@ func readMapStringString(ctx *ReadContext) map[string]string {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- // Handle null key/value cases
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull && valueHasNull {
- // Both null - use empty strings for key and value
- result[""] = ""
- size--
- continue
- } else if keyHasNull {
- // Null key with non-null value
- valueDeclared := (chunkHeader & VALUE_DECL_TYPE) != 0
- if !valueDeclared {
- buf.ReadUint8(err) // skip value type
- }
- v := readString(buf, err)
- result[""] = v // empty string as null key
- size--
- continue
- } else if valueHasNull {
- // Non-null key with null value
- keyDeclared := (chunkHeader & KEY_DECL_TYPE) != 0
- if !keyDeclared {
- buf.ReadUint8(err) // skip key type
- }
- k := readString(buf, err)
- result[k] = "" // empty string as null value
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
- // ReadData chunk size
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
- // Read type info if not DECL_TYPE
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err) // skip key type
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err) // skip value type
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
- // ReadData chunk entries
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := readString(buf, err)
result[k] = v
@@ -185,22 +167,30 @@ func readMapStringInt64(ctx *ReadContext) map[string]int64 {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := buf.ReadVarint64(err)
result[k] = v
@@ -261,22 +251,30 @@ func readMapStringInt32(ctx *ReadContext) map[string]int32 {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT32) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := buf.ReadVarint32(err)
result[k] = v
@@ -337,22 +335,30 @@ func readMapStringInt(ctx *ReadContext) map[string]int {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := buf.ReadVarint64(err)
result[k] = int(v)
@@ -413,22 +419,30 @@ func readMapStringFloat64(ctx *ReadContext) map[string]float64 {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(FLOAT64) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := buf.ReadFloat64(err)
result[k] = v
@@ -489,27 +503,34 @@ func readMapStringBool(ctx *ReadContext) map[string]bool {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
- // Read type info (written by writeMapStringBool)
keyDeclType := (chunkHeader & KEY_DECL_TYPE) != 0
valDeclType := (chunkHeader & VALUE_DECL_TYPE) != 0
if !keyDeclType {
- buf.ReadUint8(err) // skip key type info
+ if !ctx.readExpectedTypeID(STRING) {
+ return result
+ }
}
if !valDeclType {
- buf.ReadUint8(err) // skip value type info
+ if !ctx.readExpectedTypeID(BOOL) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := readString(buf, err)
v := buf.ReadBool(err)
result[k] = v
@@ -570,22 +591,30 @@ func readMapInt32Int32(ctx *ReadContext) map[int32]int32 {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT32) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT32) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := buf.ReadVarint32(err)
v := buf.ReadVarint32(err)
result[k] = v
@@ -646,22 +675,30 @@ func readMapInt64Int64(ctx *ReadContext) map[int64]int64 {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := buf.ReadVarint64(err)
v := buf.ReadVarint64(err)
result[k] = v
@@ -722,22 +759,30 @@ func readMapIntInt(ctx *ReadContext) map[int]int {
for size > 0 {
chunkHeader := buf.ReadUint8(err)
- keyHasNull := (chunkHeader & KEY_HAS_NULL) != 0
- valueHasNull := (chunkHeader & VALUE_HAS_NULL) != 0
-
- if keyHasNull || valueHasNull {
- size--
- continue
+ if chunkHeader&(TRACKING_KEY_REF|KEY_HAS_NULL|TRACKING_VALUE_REF|VALUE_HAS_NULL) != 0 {
+ ctx.SetError(DeserializationError("typed map reader does not support ref/null chunks"))
+ return result
}
chunkSize := int(buf.ReadUint8(err))
+ if ctx.HasError() {
+ return result
+ }
+ if chunkSize == 0 || chunkSize > size {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, size))
+ return result
+ }
if (chunkHeader & KEY_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
if (chunkHeader & VALUE_DECL_TYPE) == 0 {
- buf.ReadUint8(err)
+ if !ctx.readExpectedTypeID(VARINT64) {
+ return result
+ }
}
- for i := 0; i < chunkSize && size > 0; i++ {
+ for i := 0; i < chunkSize; i++ {
k := buf.ReadVarint64(err)
v := buf.ReadVarint64(err)
result[int(k)] = int(v)
diff --git a/go/fory/map_primitive_test.go b/go/fory/map_primitive_test.go
new file mode 100644
index 000000000..7decca8b0
--- /dev/null
+++ b/go/fory/map_primitive_test.go
@@ -0,0 +1,61 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fory
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestPrimitiveMapReaderRejectsInvalidChunkSize(t *testing.T) {
+ f := NewFory()
+ buf := NewByteBuffer(nil)
+ buf.WriteLength(1)
+ buf.WriteUint8(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteUint8(2)
+
+ f.readCtx.SetData(buf.Bytes())
+ _ = f.readCtx.ReadStringStringMap(RefModeNone, false)
+ require.Error(t, f.readCtx.CheckError())
+}
+
+func TestPrimitiveMapReaderRejectsUnexpectedTypeInfo(t *testing.T) {
+ f := NewFory()
+ buf := NewByteBuffer(nil)
+ buf.WriteLength(1)
+ buf.WriteUint8(0)
+ buf.WriteUint8(1)
+ buf.WriteUint8(uint8(STRING))
+ buf.WriteUint8(uint8(BOOL))
+
+ f.readCtx.SetData(buf.Bytes())
+ _ = f.readCtx.ReadStringStringMap(RefModeNone, false)
+ require.Error(t, f.readCtx.CheckError())
+}
+
+func TestPrimitiveMapReaderRejectsNullChunks(t *testing.T) {
+ f := NewFory()
+ buf := NewByteBuffer(nil)
+ buf.WriteLength(1)
+ buf.WriteUint8(KEY_HAS_NULL)
+
+ f.readCtx.SetData(buf.Bytes())
+ _ = f.readCtx.ReadStringStringMap(RefModeNone, false)
+ require.Error(t, f.readCtx.CheckError())
+}
diff --git a/go/fory/meta_string_resolver.go b/go/fory/meta_string_resolver.go
index 6df61d0c7..5a9e1cb8f 100644
--- a/go/fory/meta_string_resolver.go
+++ b/go/fory/meta_string_resolver.go
@@ -18,7 +18,6 @@
package fory
import (
- "bytes"
"encoding/binary"
"fmt"
"github.com/apache/fory/go/fory/meta"
@@ -28,10 +27,10 @@ import (
const (
SmallStringThreshold = 16 // Maximum length for "small" strings
DefaultDynamicWriteMetaStrID = -1 // Default ID for dynamic strings
+ maxCachedMetaStrings = 8192
+ smallMetaStringEncodingBits = 4
)
-type Encoding int8
-
type MetaStringBytes struct {
Data []byte
Length int16
@@ -60,14 +59,22 @@ func (a *MetaStringBytes) Hash() int64 {
type pair [2]int64
+// Mirrors Java's small MetaString read cache key: two packed byte words plus one
+// compact length/encoding byte. The packed words are zero-padded and are not
+// exact byte identity by themselves.
+type smallMetaStringKey struct {
+ v1 int64
+ v2 int64
+ compactKey byte
+}
+
type MetaStringResolver struct {
- dynamicWriteStringID int16 // Counter for dynamic string IDs
- dynamicWrittenEnumString []*MetaStringBytes // Cache of written strings
- dynamicIDToEnumString []*MetaStringBytes // Cache of read strings by ID
- hashToMetaStrBytes map[int64]*MetaStringBytes // Large string lookup
- smallHashToMetaStrBytes map[pair]*MetaStringBytes // Small string lookup
- enumStrSet map[*MetaStringBytes]struct{} // String set for deduplication
- metaStrToMetaStrBytes map[*meta.MetaString]*MetaStringBytes // Conversion cache
+ dynamicWriteStringID int16 // Counter for dynamic string IDs
+ dynamicWrittenEnumString []*MetaStringBytes // Cache of written strings
+ dynamicIDToEnumString []*MetaStringBytes // Cache of read strings by ID
+ hashToMetaStrBytes map[int64]*MetaStringBytes // Large string lookup
+ smallHashToMetaStrBytes map[smallMetaStringKey]*MetaStringBytes // Small string lookup
+ metaStrToMetaStrBytes map[*meta.MetaString]*MetaStringBytes // Conversion cache
}
var emptyMetaStringBytes = NewMetaStringBytes([]byte{}, 256)
@@ -75,8 +82,7 @@ var emptyMetaStringBytes = NewMetaStringBytes([]byte{}, 256)
func NewMetaStringResolver() *MetaStringResolver {
return &MetaStringResolver{
hashToMetaStrBytes: make(map[int64]*MetaStringBytes),
- smallHashToMetaStrBytes: make(map[pair]*MetaStringBytes),
- enumStrSet: make(map[*MetaStringBytes]struct{}),
+ smallHashToMetaStrBytes: make(map[smallMetaStringKey]*MetaStringBytes),
metaStrToMetaStrBytes: make(map[*meta.MetaString]*MetaStringBytes),
}
}
@@ -121,20 +127,27 @@ func (r *MetaStringResolver) ReadMetaStringBytes(buf *ByteBuffer, ctxErr *Error)
return nil, *ctxErr
}
- length := int16(header >> 1)
+ lengthValue := header >> 1
if header&1 != 0 {
- index := int(length) - 1
+ if lengthValue == 0 || uint64(lengthValue) > uint64(MaxInt) {
+ return nil, fmt.Errorf("invalid dynamic index: %d", lengthValue)
+ }
+ index := int(lengthValue) - 1
if index < 0 || index >= len(r.dynamicIDToEnumString) {
return nil, fmt.Errorf("invalid dynamic index: %d", index)
}
return r.dynamicIDToEnumString[index], nil
}
+ if lengthValue > uint32(MaxInt16) {
+ return nil, fmt.Errorf("meta string length %d exceeds maximum supported length %d", lengthValue, MaxInt16)
+ }
+ length := int(lengthValue)
var (
hashcode int64
- key pair
+ key smallMetaStringKey
data []byte
- encoding Encoding
+ encoding meta.Encoding
)
// Small string optimization
@@ -143,9 +156,12 @@ func (r *MetaStringResolver) ReadMetaStringBytes(buf *ByteBuffer, ctxErr *Error)
r.dynamicIDToEnumString = append(r.dynamicIDToEnumString, emptyMetaStringBytes)
return emptyMetaStringBytes, nil
}
- // ReadData encoding and data
encByte := buf.ReadByte(ctxErr)
- encoding = Encoding(encByte)
+ var encErr error
+ encoding, encErr = meta.EncodingFromByte(encByte)
+ if encErr != nil {
+ return nil, encErr
+ }
data = make([]byte, length)
_, err := buf.Read(data)
@@ -153,29 +169,33 @@ func (r *MetaStringResolver) ReadMetaStringBytes(buf *ByteBuffer, ctxErr *Error)
return nil, err
}
- // Compute composite hash key
- if length <= 8 {
- key[0] = bytesToInt64(data)
- } else {
- err := binary.Read(bytes.NewReader(data[:8]), binary.LittleEndian, &key[0])
- if err != nil {
- return nil, err
- }
- key[1] = bytesToInt64(data[8:])
+ words := smallMetaStringWords(data)
+ key = smallMetaStringKey{
+ v1: words[0],
+ v2: words[1],
+ compactKey: byte(((length - 1) << smallMetaStringEncodingBits) | int(encoding)),
}
- hashcode = ((key[0]*31 + key[1]) >> 8 << 8) | int64(encoding)
+ hashcode = computeSmallMetaStringHash(words, length, encoding)
} else {
// Large string handling
err := binary.Read(buf, binary.LittleEndian, &hashcode)
if err != nil {
return nil, err
}
- encoding = Encoding(hashcode & 0xFF)
+ var encErr error
+ encoding, encErr = meta.EncodingFromByte(byte(hashcode & 0xFF))
+ if encErr != nil {
+ return nil, encErr
+ }
data = make([]byte, length)
_, err = buf.Read(data)
if err != nil {
return nil, err
}
+ canonicalHashcode := ComputeMetaStringHash(data, encoding)
+ if canonicalHashcode != hashcode {
+ return nil, fmt.Errorf("meta string body hash mismatch")
+ }
}
// Check string caches for existing instance
@@ -191,14 +211,18 @@ func (r *MetaStringResolver) ReadMetaStringBytes(buf *ByteBuffer, ctxErr *Error)
}
}
- // Create and cache new string instance
+ // Cache only after the current body has been parsed and, for large bodies, hash-validated.
+ // Header-keyed hits stay on the fast path; forged headers cannot poison the shared cache.
m := NewMetaStringBytes(data, hashcode)
if length <= SmallStringThreshold {
- r.smallHashToMetaStrBytes[key] = m
+ if len(r.smallHashToMetaStrBytes) < maxCachedMetaStrings {
+ r.smallHashToMetaStrBytes[key] = m
+ }
} else {
- r.hashToMetaStrBytes[hashcode] = m
+ if len(r.hashToMetaStrBytes) < maxCachedMetaStrings {
+ r.hashToMetaStrBytes[hashcode] = m
+ }
}
- r.enumStrSet[m] = struct{}{}
r.dynamicIDToEnumString = append(r.dynamicIDToEnumString, m)
return m, nil
@@ -222,14 +246,8 @@ func (r *MetaStringResolver) GetMetaStrBytes(metastr *meta.MetaString) *MetaStri
}
if length <= SmallStringThreshold {
// Small string: use direct bytes as hash components
- var v1, v2 int64
- if length <= 8 {
- v1 = bytesToInt64(data)
- } else {
- binary.Read(bytes.NewReader(data[:8]), binary.LittleEndian, &v1)
- v2 = bytesToInt64(data[8:])
- }
- hashcode = ((v1*31 + v2) >> 8 << 8) | int64(metastr.GetEncoding())
+ words := smallMetaStringWords(data)
+ hashcode = computeSmallMetaStringHash(words, length, metastr.GetEncoding())
} else {
// Large string: use MurmurHash3
h64 := Murmur3Sum64WithSeed(data, 47)
@@ -253,14 +271,8 @@ func ComputeMetaStringHash(data []byte, encoding meta.Encoding) int64 {
hashcode |= int64(encoding)
} else if length <= SmallStringThreshold {
// Small string: use direct bytes as hash components
- var v1, v2 int64
- if length <= 8 {
- v1 = bytesToInt64(data)
- } else {
- binary.Read(bytes.NewReader(data[:8]), binary.LittleEndian, &v1)
- v2 = bytesToInt64(data[8:])
- }
- hashcode = ((v1*31 + v2) >> 8 << 8) | int64(encoding)
+ words := smallMetaStringWords(data)
+ hashcode = computeSmallMetaStringHash(words, length, encoding)
} else {
// Large string: use MurmurHash3
h64 := Murmur3Sum64WithSeed(data, 47)
@@ -316,3 +328,15 @@ func bytesToInt64(b []byte) int64 {
}
return v
}
+
+func smallMetaStringWords(data []byte) pair {
+ if len(data) <= 8 {
+ return pair{bytesToInt64(data), 0}
+ }
+ return pair{int64(binary.LittleEndian.Uint64(data[:8])), bytesToInt64(data[8:])}
+}
+
+func computeSmallMetaStringHash(words pair, length int, encoding meta.Encoding) int64 {
+ hash := uint64(words[0]*31+words[1]) ^ (uint64(length) << 56)
+ return int64((hash & 0xffffffffffffff00) | uint64(encoding))
+}
diff --git a/go/fory/meta_string_resolver_test.go b/go/fory/meta_string_resolver_test.go
index bbf9e3902..5a15be802 100644
--- a/go/fory/meta_string_resolver_test.go
+++ b/go/fory/meta_string_resolver_test.go
@@ -18,8 +18,11 @@
package fory
import (
+ "encoding/binary"
"github.com/stretchr/testify/require"
"testing"
+
+ "github.com/apache/fory/go/fory/meta"
)
// TestMetaStringResolverNegativeIndexPanic reproduces the CRITICAL security bug
@@ -65,3 +68,125 @@ func TestMetaStringResolverBoundaryRegression(t *testing.T) {
require.NoError(t, err)
require.Equal(t, m, result, "Should correctly resolve the first dynamic string (index 0)")
}
+
+func TestMetaStringResolverRejectsLargeBodyHashMismatch(t *testing.T) {
+ resolver := NewMetaStringResolver()
+ buffer := NewByteBuffer(nil)
+ data := []byte("0123456789abcdefg")
+
+ buffer.WriteVarUint32Small7(uint32(len(data)) << 1)
+ buffer.WriteInt64(int64(meta.UTF_8))
+ buffer.Write(data)
+ buffer.SetReaderIndex(0)
+
+ var ctxErr Error
+ _, err := resolver.ReadMetaStringBytes(buffer, &ctxErr)
+ require.Error(t, err)
+ require.Empty(t, resolver.hashToMetaStrBytes)
+ require.Empty(t, resolver.dynamicIDToEnumString)
+}
+
+func TestMetaStringResolverRejectsOversizedLengthBeforeAllocation(t *testing.T) {
+ resolver := NewMetaStringResolver()
+ buffer := NewByteBuffer(nil)
+ buffer.WriteVarUint32Small7(uint32(MaxInt16+1) << 1)
+ buffer.SetReaderIndex(0)
+
+ var ctxErr Error
+ _, err := resolver.ReadMetaStringBytes(buffer, &ctxErr)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "meta string length")
+ require.Empty(t, resolver.hashToMetaStrBytes)
+ require.Empty(t, resolver.smallHashToMetaStrBytes)
+ require.Empty(t, resolver.dynamicIDToEnumString)
+}
+
+func TestMetaStringResolverSmallCacheKeyIncludesLengthAndEncoding(t *testing.T) {
+ resolver := NewMetaStringResolver()
+
+ oneByte := NewByteBuffer(nil)
+ oneByte.WriteVarUint32Small7(1 << 1)
+ oneByte.WriteByte(byte(meta.UTF_8))
+ oneByte.WriteByte(1)
+ oneByte.SetReaderIndex(0)
+ var oneErr Error
+ first, err := resolver.ReadMetaStringBytes(oneByte, &oneErr)
+ require.NoError(t, err)
+ require.Equal(t, []byte{1}, first.Data)
+
+ twoBytes := NewByteBuffer(nil)
+ twoBytes.WriteVarUint32Small7(2 << 1)
+ twoBytes.WriteByte(byte(meta.UTF_8))
+ twoBytes.Write([]byte{1, 0})
+ twoBytes.SetReaderIndex(0)
+ var twoErr Error
+ second, err := resolver.ReadMetaStringBytes(twoBytes, &twoErr)
+ require.NoError(t, err)
+ require.Equal(t, []byte{1, 0}, second.Data)
+ require.NotSame(t, first, second)
+
+ differentEncoding := NewByteBuffer(nil)
+ differentEncoding.WriteVarUint32Small7(1 << 1)
+ differentEncoding.WriteByte(byte(meta.LOWER_SPECIAL))
+ differentEncoding.WriteByte(1)
+ differentEncoding.SetReaderIndex(0)
+ var encodingErr Error
+ third, err := resolver.ReadMetaStringBytes(differentEncoding, &encodingErr)
+ require.NoError(t, err)
+ require.Equal(t, []byte{1}, third.Data)
+ require.Equal(t, meta.LOWER_SPECIAL, third.Encoding)
+ require.NotSame(t, first, third)
+ require.Len(t, resolver.smallHashToMetaStrBytes, 3)
+}
+
+func TestComputeMetaStringHashIncludesSmallLength(t *testing.T) {
+ require.NotEqual(
+ t,
+ ComputeMetaStringHash([]byte{1}, meta.UTF_8),
+ ComputeMetaStringHash([]byte{1, 0}, meta.UTF_8),
+ )
+ require.NotEqual(
+ t,
+ ComputeMetaStringHash([]byte{1}, meta.UTF_8),
+ ComputeMetaStringHash([]byte{1}, meta.LOWER_SPECIAL),
+ )
+}
+
+func TestMetaStringResolverReadCachesAreCapped(t *testing.T) {
+ resolver := NewMetaStringResolver()
+ smallKey := smallMetaStringKey{v1: 1, v2: 0, compactKey: byte(meta.UTF_8)}
+ for i := 0; i < maxCachedMetaStrings; i++ {
+ resolver.smallHashToMetaStrBytes[smallMetaStringKey{
+ v1: int64(i + 2),
+ v2: 0,
+ compactKey: byte(meta.UTF_8),
+ }] =
+ NewMetaStringBytes([]byte{byte(i)}, int64(i+2)<<8)
+ resolver.hashToMetaStrBytes[int64(i+2)<<8] =
+ NewMetaStringBytes([]byte("0123456789abcdefg"), int64(i+2)<<8)
+ }
+
+ smallBuffer := NewByteBuffer(nil)
+ smallBuffer.WriteVarUint32Small7(1 << 1)
+ smallBuffer.WriteByte(byte(meta.UTF_8))
+ smallBuffer.WriteByte(1)
+ smallBuffer.SetReaderIndex(0)
+ var smallErr Error
+ _, err := resolver.ReadMetaStringBytes(smallBuffer, &smallErr)
+ require.NoError(t, err)
+ require.Len(t, resolver.smallHashToMetaStrBytes, maxCachedMetaStrings)
+ require.NotContains(t, resolver.smallHashToMetaStrBytes, smallKey)
+
+ largeData := []byte("0123456789abcdefg")
+ largeHash := ComputeMetaStringHash(largeData, meta.UTF_8)
+ largeBuffer := NewByteBuffer(nil)
+ largeBuffer.WriteVarUint32Small7(uint32(len(largeData)) << 1)
+ require.NoError(t, binary.Write(largeBuffer, binary.LittleEndian, largeHash))
+ largeBuffer.Write(largeData)
+ largeBuffer.SetReaderIndex(0)
+ var largeErr Error
+ _, err = resolver.ReadMetaStringBytes(largeBuffer, &largeErr)
+ require.NoError(t, err)
+ require.Len(t, resolver.hashToMetaStrBytes, maxCachedMetaStrings)
+ require.NotContains(t, resolver.hashToMetaStrBytes, largeHash)
+}
diff --git a/go/fory/primitive.go b/go/fory/primitive.go
index f5a7e4055..1748bc27a 100644
--- a/go/fory/primitive.go
+++ b/go/fory/primitive.go
@@ -56,8 +56,8 @@ func (s boolSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(BOOL) {
+ return
}
if ctx.HasError() {
return
@@ -103,8 +103,8 @@ func (s int8Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(INT8) {
+ return
}
if ctx.HasError() {
return
@@ -149,8 +149,8 @@ func (s byteSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(UINT8) {
+ return
}
if ctx.HasError() {
return
@@ -195,8 +195,8 @@ func (s uint16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(UINT16) {
+ return
}
if ctx.HasError() {
return
@@ -241,8 +241,8 @@ func (s uint32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VAR_UINT32) {
+ return
}
if ctx.HasError() {
return
@@ -287,8 +287,8 @@ func (s uint64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VAR_UINT64) {
+ return
}
if ctx.HasError() {
return
@@ -331,8 +331,8 @@ func (s uintSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VAR_UINT64) {
+ return
}
if ctx.HasError() {
return
@@ -375,8 +375,8 @@ func (s int16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(INT16) {
+ return
}
if ctx.HasError() {
return
@@ -419,8 +419,8 @@ func (s int32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VARINT32) {
+ return
}
if ctx.HasError() {
return
@@ -463,8 +463,8 @@ func (s int64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VARINT64) {
+ return
}
if ctx.HasError() {
return
@@ -505,8 +505,8 @@ func (s intSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, ha
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(VARINT64) {
+ return
}
if ctx.HasError() {
return
@@ -549,8 +549,8 @@ func (s float32Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(FLOAT32) {
+ return
}
if ctx.HasError() {
return
@@ -593,8 +593,8 @@ func (s float64Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(FLOAT64) {
+ return
}
if ctx.HasError() {
return
@@ -651,8 +651,8 @@ func (s float16Serializer) Read(ctx *ReadContext, refMode RefMode, readType bool
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(FLOAT16) {
+ return
}
if ctx.HasError() {
return
@@ -704,8 +704,8 @@ func (s bfloat16Serializer) Read(ctx *ReadContext, refMode RefMode, readType boo
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(BFLOAT16) {
+ return
}
if ctx.HasError() {
return
diff --git a/go/fory/reader.go b/go/fory/reader.go
index 9c8b049ad..d0f89fa81 100644
--- a/go/fory/reader.go
+++ b/go/fory/reader.go
@@ -31,11 +31,12 @@ import (
type ReadContext struct {
buffer *ByteBuffer
refReader *RefReader
- trackRef bool // Cached flag to avoid indirection
- xlang bool // Cross-language serialization mode
+ trackRef bool // Cached flag to avoid indirection
+ xlang bool // Cross-language serialization mode
+ rootHeader byte
compatible bool // Schema evolution compatibility mode
typeResolver *TypeResolver // For complex type deserialization
- refResolver *RefResolver // For reference tracking (legacy)
+ refResolver *RefResolver // For reference tracking in native-mode paths
outOfBandBuffers []*ByteBuffer // Out-of-band buffers for deserialization
outOfBandIndex int // Current index into out-of-band buffers
depth int // Current nesting depth for cycle detection
@@ -109,7 +110,7 @@ func (c *ReadContext) TypeResolver() *TypeResolver {
return c.typeResolver
}
-// RefResolver returns the reference resolver (legacy)
+// RefResolver returns the reference resolver.
func (c *ReadContext) RefResolver() *RefResolver {
return c.refResolver
}
@@ -151,6 +152,18 @@ func (c *ReadContext) CheckError() error {
return nil
}
+func (c *ReadContext) readExpectedTypeID(expected TypeId) bool {
+ actual := TypeId(c.buffer.ReadUint8(c.Err()))
+ if c.HasError() {
+ return false
+ }
+ if actual != expected {
+ c.SetError(TypeMismatchError(actual, expected))
+ return false
+ }
+ return true
+}
+
// Inline primitive reads
func (c *ReadContext) RawBool() bool { return c.buffer.ReadBool(c.Err()) }
func (c *ReadContext) RawInt8() int8 { return int8(c.buffer.ReadByte(c.Err())) }
@@ -288,7 +301,11 @@ func (c *ReadContext) ReadBoolSlice(refMode RefMode, readType bool) []bool {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != BOOL_ARRAY {
+ c.SetError(TypeMismatchError(actual, BOOL_ARRAY))
+ return nil
+ }
}
return ReadBoolSlice(c.buffer, err)
}
@@ -302,7 +319,11 @@ func (c *ReadContext) ReadInt8Slice(refMode RefMode, readType bool) []int8 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != INT8_ARRAY {
+ c.SetError(TypeMismatchError(actual, INT8_ARRAY))
+ return nil
+ }
}
return ReadInt8Slice(c.buffer, err)
}
@@ -316,7 +337,11 @@ func (c *ReadContext) ReadInt16Slice(refMode RefMode, readType bool) []int16 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != INT16_ARRAY {
+ c.SetError(TypeMismatchError(actual, INT16_ARRAY))
+ return nil
+ }
}
return ReadInt16Slice(c.buffer, err)
}
@@ -330,7 +355,11 @@ func (c *ReadContext) ReadInt32Slice(refMode RefMode, readType bool) []int32 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != INT32_ARRAY {
+ c.SetError(TypeMismatchError(actual, INT32_ARRAY))
+ return nil
+ }
}
return ReadInt32Slice(c.buffer, err)
}
@@ -344,7 +373,11 @@ func (c *ReadContext) ReadInt64Slice(refMode RefMode, readType bool) []int64 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != INT64_ARRAY {
+ c.SetError(TypeMismatchError(actual, INT64_ARRAY))
+ return nil
+ }
}
return ReadInt64Slice(c.buffer, err)
}
@@ -358,7 +391,11 @@ func (c *ReadContext) ReadUint16Slice(refMode RefMode, readType bool) []uint16 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != UINT16_ARRAY {
+ c.SetError(TypeMismatchError(actual, UINT16_ARRAY))
+ return nil
+ }
}
return ReadUint16Slice(c.buffer, err)
}
@@ -372,7 +409,11 @@ func (c *ReadContext) ReadUint32Slice(refMode RefMode, readType bool) []uint32 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != UINT32_ARRAY {
+ c.SetError(TypeMismatchError(actual, UINT32_ARRAY))
+ return nil
+ }
}
return ReadUint32Slice(c.buffer, err)
}
@@ -386,7 +427,11 @@ func (c *ReadContext) ReadUint64Slice(refMode RefMode, readType bool) []uint64 {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != UINT64_ARRAY {
+ c.SetError(TypeMismatchError(actual, UINT64_ARRAY))
+ return nil
+ }
}
return ReadUint64Slice(c.buffer, err)
}
@@ -400,7 +445,15 @@ func (c *ReadContext) ReadIntSlice(refMode RefMode, readType bool) []int {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ expected := TypeId(INT64_ARRAY)
+ if strconv.IntSize == 32 {
+ expected = INT32_ARRAY
+ }
+ if actual != expected {
+ c.SetError(TypeMismatchError(actual, expected))
+ return nil
+ }
}
return ReadIntSlice(c.buffer, err)
}
@@ -414,7 +467,15 @@ func (c *ReadContext) ReadUintSlice(refMode RefMode, readType bool) []uint {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ expected := TypeId(UINT64_ARRAY)
+ if strconv.IntSize == 32 {
+ expected = UINT32_ARRAY
+ }
+ if actual != expected {
+ c.SetError(TypeMismatchError(actual, expected))
+ return nil
+ }
}
return ReadUintSlice(c.buffer, err)
}
@@ -428,7 +489,11 @@ func (c *ReadContext) ReadFloat32Slice(refMode RefMode, readType bool) []float32
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != FLOAT32_ARRAY {
+ c.SetError(TypeMismatchError(actual, FLOAT32_ARRAY))
+ return nil
+ }
}
return ReadFloat32Slice(c.buffer, err)
}
@@ -442,7 +507,11 @@ func (c *ReadContext) ReadFloat64Slice(refMode RefMode, readType bool) []float64
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != FLOAT64_ARRAY {
+ c.SetError(TypeMismatchError(actual, FLOAT64_ARRAY))
+ return nil
+ }
}
return ReadFloat64Slice(c.buffer, err)
}
@@ -456,7 +525,11 @@ func (c *ReadContext) ReadByteSlice(refMode RefMode, readType bool) []byte {
}
}
if readType {
- _ = c.buffer.ReadUint8(err)
+ actual := TypeId(c.buffer.ReadUint8(err))
+ if actual != BINARY && actual != UINT8_ARRAY {
+ c.SetError(DeserializationErrorf("slice type mismatch: expected BINARY (%d) or UINT8_ARRAY (%d), got %d", BINARY, UINT8_ARRAY, actual))
+ return nil
+ }
}
size := c.ReadBinaryLength()
return c.buffer.ReadBinary(size, err)
@@ -484,8 +557,8 @@ func (c *ReadContext) ReadStringStringMap(refMode RefMode, readType bool) map[st
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringString(c)
}
@@ -498,8 +571,8 @@ func (c *ReadContext) ReadStringInt64Map(refMode RefMode, readType bool) map[str
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringInt64(c)
}
@@ -512,8 +585,8 @@ func (c *ReadContext) ReadStringInt32Map(refMode RefMode, readType bool) map[str
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringInt32(c)
}
@@ -526,8 +599,8 @@ func (c *ReadContext) ReadStringIntMap(refMode RefMode, readType bool) map[strin
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringInt(c)
}
@@ -540,8 +613,8 @@ func (c *ReadContext) ReadStringFloat64Map(refMode RefMode, readType bool) map[s
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringFloat64(c)
}
@@ -554,8 +627,8 @@ func (c *ReadContext) ReadStringBoolMap(refMode RefMode, readType bool) map[stri
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapStringBool(c)
}
@@ -568,8 +641,8 @@ func (c *ReadContext) ReadInt32Int32Map(refMode RefMode, readType bool) map[int3
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapInt32Int32(c)
}
@@ -582,8 +655,8 @@ func (c *ReadContext) ReadInt64Int64Map(refMode RefMode, readType bool) map[int6
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapInt64Int64(c)
}
@@ -596,8 +669,8 @@ func (c *ReadContext) ReadIntIntMap(refMode RefMode, readType bool) map[int]int
return nil
}
}
- if readType {
- _ = c.buffer.ReadUint8(err)
+ if readType && !c.readExpectedTypeID(MAP) {
+ return nil
}
return readMapIntInt(c)
}
diff --git a/go/fory/skip.go b/go/fory/skip.go
index 9d08f465c..abc746644 100644
--- a/go/fory/skip.go
+++ b/go/fory/skip.go
@@ -56,7 +56,10 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag
// Check if it's an EXT type first - EXT types don't have meta info like structs
if internalID == EXT {
- typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err)
+ typeInfo := readKnownTypeInfoForSkip(ctx, wroteTypeID)
+ if ctx.HasError() {
+ return
+ }
if typeInfo != nil && typeInfo.Serializer != nil {
// Use the serializer to read and discard the value
var dummy any
@@ -71,8 +74,11 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag
// Check if it's a NAMED_EXT type - need to read type info to find serializer
if internalID == NAMED_EXT {
- typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err)
- if typeInfo.Serializer != nil {
+ typeInfo := readKnownTypeInfoForSkip(ctx, wroteTypeID)
+ if ctx.HasError() {
+ return
+ }
+ if typeInfo != nil && typeInfo.Serializer != nil {
// Use the serializer to read and discard the value
var dummy any
dummyVal := reflect.ValueOf(&dummy).Elem()
@@ -86,14 +92,20 @@ func SkipFieldValueWithTypeFlag(ctx *ReadContext, fieldDef FieldDef, readRefFlag
// Check if it's a struct type - need to read type info and skip struct data
if internalID == COMPATIBLE_STRUCT || internalID == STRUCT ||
internalID == NAMED_STRUCT || internalID == NAMED_COMPATIBLE_STRUCT {
- typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err)
+ typeInfo := readKnownTypeInfoForSkip(ctx, wroteTypeID)
+ if ctx.HasError() {
+ return
+ }
// Now skip the struct data using the typeInfo from the written type
skipStruct(ctx, typeInfo)
return
}
if IsNamespacedType(internalID) {
- typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, wroteTypeID, err)
+ typeInfo := readKnownTypeInfoForSkip(ctx, wroteTypeID)
+ if ctx.HasError() {
+ return
+ }
// Now skip the struct data using the typeInfo from the written type
skipStruct(ctx, typeInfo)
return
@@ -153,38 +165,22 @@ func SkipAnyValue(ctx *ReadContext, readRefFlag bool) {
typeSpec: NewMapTypeSpec(TypeId(typeID), NewSimpleTypeSpec(UNKNOWN), NewSimpleTypeSpec(UNKNOWN)),
nullable: true,
}
- case NAMED_UNION:
- resolver := ctx.TypeResolver()
- _, _ = resolver.metaStringResolver.ReadMetaStringBytes(ctx.buffer, err)
- if ctx.HasError() {
- return
- }
- _, _ = resolver.metaStringResolver.ReadMetaStringBytes(ctx.buffer, err)
- if ctx.HasError() {
- return
- }
- fieldDef = FieldDef{
- typeSpec: NewSimpleTypeSpec(TypeId(typeID)),
- nullable: true,
- }
- case COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, STRUCT, NAMED_STRUCT, EXT, TYPED_UNION:
+ case ENUM, NAMED_ENUM, COMPATIBLE_STRUCT, NAMED_COMPATIBLE_STRUCT, STRUCT, NAMED_STRUCT,
+ EXT, NAMED_EXT, TYPED_UNION, NAMED_UNION:
// Read type info using the shared meta reader when enabled.
typeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err)
if ctx.HasError() {
return
}
+ if typeInfo == nil {
+ ctx.SetError(DeserializationErrorf("cannot skip type %d: type info not found", typeID))
+ return
+ }
fieldDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(typeID)),
nullable: true,
}
default:
- if internalID == ENUM || internalID == STRUCT ||
- internalID == EXT || internalID == TYPED_UNION {
- ctx.buffer.ReadVarUint32(err)
- if ctx.HasError() {
- return
- }
- }
fieldDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(typeID)),
nullable: true,
@@ -206,7 +202,25 @@ func readTypeInfoForSkip(ctx *ReadContext, fieldTypeId TypeId) *TypeInfo {
return nil
}
// Use readTypeInfoWithTypeID which handles both namespaced and non-namespaced types correctly
- return ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err)
+ typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err)
+ if ctx.HasError() {
+ return nil
+ }
+ if typeInfo == nil {
+ ctx.SetError(DeserializationErrorf("cannot skip type %d: type info not found", typeID))
+ }
+ return typeInfo
+}
+
+func readKnownTypeInfoForSkip(ctx *ReadContext, typeID uint32) *TypeInfo {
+ typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, ctx.Err())
+ if ctx.HasError() {
+ return nil
+ }
+ if typeInfo == nil {
+ ctx.SetError(DeserializationErrorf("cannot skip type %d: type info not found", typeID))
+ }
+ return typeInfo
}
// skipCollection skips a collection (list/set) value
@@ -236,7 +250,10 @@ func skipCollection(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
- elemTypeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, err)
+ elemTypeInfo = readKnownTypeInfoForSkip(ctx, typeID)
+ if ctx.HasError() {
+ return
+ }
elemDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(elemTypeInfo.TypeID)),
nullable: hasNull,
@@ -336,7 +353,10 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
- valueTypeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, bufErr)
+ valueTypeInfo = readKnownTypeInfoForSkip(ctx, typeID)
+ if ctx.HasError() {
+ return
+ }
valueDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(valueTypeInfo.TypeID)),
nullable: true,
@@ -368,7 +388,10 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
- keyTypeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, bufErr)
+ keyTypeInfo = readKnownTypeInfoForSkip(ctx, typeID)
+ if ctx.HasError() {
+ return
+ }
keyDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(keyTypeInfo.TypeID)),
nullable: true,
@@ -395,6 +418,10 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
+ if chunkSize == 0 || uint32(chunkSize) > length-lenCounter {
+ ctx.SetError(DeserializationErrorf("invalid map chunk size %d for remaining length %d", chunkSize, length-lenCounter))
+ return
+ }
keyDeclared := (header & KEY_DECL_TYPE) != 0
valueDeclared := (header & VALUE_DECL_TYPE) != 0
@@ -406,7 +433,10 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
- keyTypeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, bufErr)
+ keyTypeInfo = readKnownTypeInfoForSkip(ctx, typeID)
+ if ctx.HasError() {
+ return
+ }
keyDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(keyTypeInfo.TypeID)),
nullable: true,
@@ -420,7 +450,10 @@ func skipMap(ctx *ReadContext, fieldDef FieldDef) {
if ctx.HasError() {
return
}
- valueTypeInfo = ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeID, bufErr)
+ valueTypeInfo = readKnownTypeInfoForSkip(ctx, typeID)
+ if ctx.HasError() {
+ return
+ }
valueDef = FieldDef{
typeSpec: NewSimpleTypeSpec(TypeId(valueTypeInfo.TypeID)),
nullable: true,
@@ -527,25 +560,32 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo
internalID := TypeId(typeIDNum)
// Handle struct-like types
if internalID == COMPATIBLE_STRUCT || internalID == STRUCT ||
- internalID == NAMED_STRUCT || internalID == NAMED_COMPATIBLE_STRUCT ||
- internalID == UNKNOWN {
+ internalID == NAMED_STRUCT || internalID == NAMED_COMPATIBLE_STRUCT {
// If type_info is provided (from SkipAnyValue), use skipStruct directly
if typeInfo != nil {
skipStruct(ctx, typeInfo)
return
}
// Otherwise we need to read type info
- ti := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeIDNum, err)
+ ti := readKnownTypeInfoForSkip(ctx, typeIDNum)
+ if ctx.HasError() {
+ return
+ }
skipStruct(ctx, ti)
return
}
- if internalID == ENUM {
+ if internalID == ENUM || internalID == NAMED_ENUM {
// Enum values are encoded as ordinal only (VarUint32Small7) for xlang.
- _ = ctx.buffer.ReadUint8(err)
+ _ = ctx.buffer.ReadVarUint32Small7(err)
return
}
if internalID == EXT || internalID == NAMED_EXT || internalID == TYPED_UNION || internalID == NAMED_UNION {
- typeInfo := ctx.TypeResolver().readTypeInfoWithTypeID(ctx.buffer, typeIDNum, err)
+ if typeInfo == nil {
+ typeInfo = readKnownTypeInfoForSkip(ctx, typeIDNum)
+ if ctx.HasError() {
+ return
+ }
+ }
if typeInfo != nil && typeInfo.Serializer != nil {
// Use the serializer to read and discard the value
var dummy any
@@ -569,11 +609,15 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo
case INT16:
_ = ctx.buffer.ReadInt16(err)
case INT32:
- _ = ctx.buffer.ReadUint8(err)
+ _ = ctx.buffer.ReadInt32(err)
case VARINT32:
- _ = ctx.buffer.ReadUint8(err)
- case INT64, VARINT64, TAGGED_INT64:
+ _ = ctx.buffer.ReadVarint32(err)
+ case INT64:
+ _ = ctx.buffer.ReadInt64(err)
+ case VARINT64:
_ = ctx.buffer.ReadVarint64(err)
+ case TAGGED_INT64:
+ _ = ctx.buffer.ReadTaggedInt64(err)
// Floating point types
case BFLOAT16, FLOAT16:
@@ -676,7 +720,7 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo
// Enum types
case ENUM:
- _ = ctx.buffer.ReadVarUint32(err)
+ _ = ctx.buffer.ReadVarUint32Small7(err)
// Unsigned integer types
case UINT8:
@@ -692,12 +736,7 @@ func skipValue(ctx *ReadContext, fieldDef FieldDef, readRefFlag bool, isField bo
case VAR_UINT64:
_ = ctx.buffer.ReadVarUint64(err)
case TAGGED_UINT64:
- firstInt32 := ctx.buffer.ReadInt32(err)
- if (firstInt32 & 1) != 0 {
- // 9-byte encoding
- _ = ctx.buffer.ReadUint64(err)
- }
- // Otherwise it's 4-byte encoding, already read
+ _ = ctx.buffer.ReadTaggedUint64(err)
// Unknown (polymorphic) type - read type info and skip dynamically
case UNKNOWN:
diff --git a/go/fory/skip_test.go b/go/fory/skip_test.go
new file mode 100644
index 000000000..c60a40e35
--- /dev/null
+++ b/go/fory/skip_test.go
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fory
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestSkipEnumConsumesSmall7Ordinal(t *testing.T) {
+ f := New(WithXlang(true))
+ buf := NewByteBuffer(nil)
+ buf.WriteVarUint32Small7(128)
+ buf.WriteByte(0x7f)
+
+ f.readCtx.SetData(buf.Bytes())
+ skipValue(
+ f.readCtx,
+ FieldDef{typeSpec: NewSimpleTypeSpec(ENUM), nullable: true},
+ false,
+ false,
+ nil,
+ )
+ require.NoError(t, f.readCtx.CheckError())
+ require.Equal(t, 2, f.readCtx.Buffer().ReaderIndex())
+ require.Equal(t, byte(0x7f), f.readCtx.Buffer().ReadByte(f.readCtx.Err()))
+}
+
+func TestSkipPrimitiveConsumesExactEncoding(t *testing.T) {
+ tests := []struct {
+ name string
+ typeID TypeId
+ write func(*ByteBuffer)
+ }{
+ {
+ name: "int32",
+ typeID: INT32,
+ write: func(buf *ByteBuffer) { buf.WriteInt32(0x01020304) },
+ },
+ {
+ name: "varint32",
+ typeID: VARINT32,
+ write: func(buf *ByteBuffer) { buf.WriteVarint32(300) },
+ },
+ {
+ name: "int64",
+ typeID: INT64,
+ write: func(buf *ByteBuffer) { buf.WriteInt64(0x0102030405060708) },
+ },
+ {
+ name: "varint64",
+ typeID: VARINT64,
+ write: func(buf *ByteBuffer) { buf.WriteVarint64(1 << 35) },
+ },
+ {
+ name: "tagged_int64_small",
+ typeID: TAGGED_INT64,
+ write: func(buf *ByteBuffer) { buf.WriteTaggedInt64(1073741823) },
+ },
+ {
+ name: "tagged_int64_large",
+ typeID: TAGGED_INT64,
+ write: func(buf *ByteBuffer) { buf.WriteTaggedInt64(1 << 40) },
+ },
+ {
+ name: "tagged_uint64_small",
+ typeID: TAGGED_UINT64,
+ write: func(buf *ByteBuffer) { buf.WriteTaggedUint64(0x7fffffff) },
+ },
+ {
+ name: "tagged_uint64_large",
+ typeID: TAGGED_UINT64,
+ write: func(buf *ByteBuffer) { buf.WriteTaggedUint64(1 << 40) },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ f := New(WithXlang(true))
+ buf := NewByteBuffer(nil)
+ tc.write(buf)
+ wantIndex := buf.WriterIndex()
+ buf.WriteByte(0x7f)
+
+ f.readCtx.SetData(buf.Bytes())
+ skipValue(
+ f.readCtx,
+ FieldDef{typeSpec: NewSimpleTypeSpec(tc.typeID), nullable: true},
+ false,
+ false,
+ nil,
+ )
+ require.NoError(t, f.readCtx.CheckError())
+ require.Equal(t, wantIndex, f.readCtx.Buffer().ReaderIndex())
+ require.Equal(t, byte(0x7f), f.readCtx.Buffer().ReadByte(f.readCtx.Err()))
+ })
+ }
+}
+
+func TestSkipMapRejectsInvalidChunkSize(t *testing.T) {
+ f := New(WithXlang(true))
+ buf := NewByteBuffer(nil)
+ buf.WriteLength(1)
+ buf.WriteByte(KEY_DECL_TYPE | VALUE_DECL_TYPE)
+ buf.WriteByte(2)
+
+ f.readCtx.SetData(buf.Bytes())
+ skipMap(
+ f.readCtx,
+ FieldDef{
+ typeSpec: NewMapTypeSpec(MAP, NewSimpleTypeSpec(INT32), NewSimpleTypeSpec(INT32)),
+ nullable: true,
+ },
+ )
+ require.Error(t, f.readCtx.CheckError())
+}
diff --git a/go/fory/stream.go b/go/fory/stream.go
index 420a6d482..bb8668959 100644
--- a/go/fory/stream.go
+++ b/go/fory/stream.go
@@ -92,47 +92,26 @@ func (is *InputStream) Shrink() {
}
// DeserializeFromStream reads the next object from the stream into the provided value.
-// It uses a shared ReadContext for the lifetime of the InputStream, clearing
-// temporary state between calls but preserving the buffer and TypeResolver state.
+// It preserves the stream buffer while clearing root-scoped read metadata between calls.
func (f *Fory) DeserializeFromStream(is *InputStream, v any) error {
-
- // We only reset the temporary read state (like refTracker and outOfBand buffers),
- // NOT the buffer or the type mapping, which must persist.
- defer func() {
- f.readCtx.refReader.Reset()
- f.readCtx.outOfBandBuffers = nil
- f.readCtx.outOfBandIndex = 0
- f.readCtx.err = Error{}
- if f.readCtx.refResolver != nil {
- f.readCtx.refResolver.resetRead()
- }
- }()
-
- // Temporarily swap buffer
origBuffer := f.readCtx.buffer
f.readCtx.buffer = is.buffer
+ defer func() {
+ f.readCtx.buffer = origBuffer
+ f.resetReadState()
+ }()
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
- f.readCtx.buffer = origBuffer
return f.readCtx.TakeError()
}
- if isNull {
- f.readCtx.buffer = origBuffer
- return nil
- }
-
target := reflect.ValueOf(v).Elem()
f.readCtx.ReadValue(target, RefModeTracking, true)
if f.readCtx.HasError() {
- f.readCtx.buffer = origBuffer
return f.readCtx.TakeError()
}
- // Restore original buffer
- f.readCtx.buffer = origBuffer
-
return nil
}
@@ -145,15 +124,11 @@ func (f *Fory) DeserializeFromReader(r io.Reader, v any) error {
// Always reset to enforce stateless semantics.
f.readCtx.buffer.ResetWithReader(r, 0)
- isNull := readHeader(f.readCtx)
+ readHeader(f.readCtx)
if f.readCtx.HasError() {
return f.readCtx.TakeError()
}
- if isNull {
- return nil
- }
-
target := reflect.ValueOf(v).Elem()
f.readCtx.ReadValue(target, RefModeTracking, true)
if f.readCtx.HasError() {
diff --git a/go/fory/stream_test.go b/go/fory/stream_test.go
index 098254587..0c2503f3c 100644
--- a/go/fory/stream_test.go
+++ b/go/fory/stream_test.go
@@ -135,6 +135,29 @@ func TestStreamDeserializationEOF(t *testing.T) {
}
}
+func TestDeserializeFromStreamClearsReadMetadataOnError(t *testing.T) {
+ f := New(WithCompatible(true))
+ f.typeResolver.metaStringResolver.dynamicIDToEnumString =
+ append(f.typeResolver.metaStringResolver.dynamicIDToEnumString, emptyMetaStringBytes)
+ f.metaContext.readTypeInfos = append(f.metaContext.readTypeInfos, &TypeInfo{})
+
+ stream := NewInputStream(bytes.NewReader(nil))
+ var out int32
+ err := f.DeserializeFromStream(stream, &out)
+ if err == nil {
+ t.Fatal("Expected error on empty stream, got nil")
+ }
+ if len(f.typeResolver.metaStringResolver.dynamicIDToEnumString) != 0 {
+ t.Fatalf(
+ "expected stream root cleanup to clear metastring refs, got %d",
+ len(f.typeResolver.metaStringResolver.dynamicIDToEnumString),
+ )
+ }
+ if len(f.metaContext.readTypeInfos) != 0 {
+ t.Fatalf("expected stream root cleanup to clear type metadata, got %d", len(f.metaContext.readTypeInfos))
+ }
+}
+
func TestInputStreamSequential(t *testing.T) {
f := New()
// Register type in compatible mode to test Meta Sharing across sequential reads
diff --git a/go/fory/string.go b/go/fory/string.go
index 7c7a41bba..9165aaf66 100644
--- a/go/fory/string.go
+++ b/go/fory/string.go
@@ -106,7 +106,9 @@ func readUTF16LE(buf *ByteBuffer, byteCount int, err *Error) string {
func readUTF8(buf *ByteBuffer, size int, err *Error) string {
data := buf.ReadBinary(size, err)
- return string(data) // Direct UTF-8 conversion
+ // Go intentionally keeps direct string conversion here. Rust is the runtime that checks UTF-8
+ // string payloads by default; Go preserves its platform behavior for invalid byte sequences.
+ return string(data)
}
// ============================================================================
@@ -152,8 +154,8 @@ func (s stringSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool,
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(STRING) {
+ return
}
if ctx.HasError() {
return
@@ -194,8 +196,8 @@ func (s ptrToStringSerializer) Read(ctx *ReadContext, refMode RefMode, readType
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(STRING) {
+ return
}
if ctx.HasError() {
return
diff --git a/go/fory/struct.go b/go/fory/struct.go
index a609113cb..b2d718db8 100644
--- a/go/fory/struct.go
+++ b/go/fory/struct.go
@@ -1541,13 +1541,13 @@ func (s *structSerializer) ReadData(ctx *ReadContext, value reflect.Value) {
}
switch field.DispatchId {
case PrimitiveVarint32DispatchId:
- storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarint32())
+ storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarint32(err))
case PrimitiveVarint64DispatchId:
storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarint64())
case PrimitiveIntDispatchId:
storeFieldValue(field.Kind, fieldPtr, optInfo, int(buf.UnsafeReadVarint64()))
case PrimitiveVarUint32DispatchId:
- storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarUint32())
+ storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarUint32(err))
case PrimitiveVarUint64DispatchId:
storeFieldValue(field.Kind, fieldPtr, optInfo, buf.UnsafeReadVarUint64())
case PrimitiveUintDispatchId:
diff --git a/go/fory/struct_test.go b/go/fory/struct_test.go
index 9dc03d161..eb9287a0c 100644
--- a/go/fory/struct_test.go
+++ b/go/fory/struct_test.go
@@ -604,14 +604,12 @@ func TestSkipAnyValueReadsSharedTypeMeta(t *testing.T) {
f.resetReadState()
f.readCtx.SetData(buf.Bytes())
- isNull := readHeader(f.readCtx)
- require.False(t, isNull)
+ readHeader(f.readCtx)
SkipAnyValue(f.readCtx, true)
require.NoError(t, f.readCtx.CheckError())
f.resetReadState()
- isNull = readHeader(f.readCtx)
- require.False(t, isNull)
+ readHeader(f.readCtx)
var out any
f.readCtx.ReadValue(reflect.ValueOf(&out).Elem(), RefModeTracking, true)
@@ -622,6 +620,17 @@ func TestSkipAnyValueReadsSharedTypeMeta(t *testing.T) {
require.Equal(t, "ok", result.Name)
}
+func TestReadHeaderRejectsOutOfBandWithoutBuffers(t *testing.T) {
+ f := New(WithXlang(true))
+ f.readCtx.SetData([]byte{XLangFlag | OutOfBandFlag})
+
+ readHeader(f.readCtx)
+
+ err := f.readCtx.TakeError()
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "out-of-band buffers")
+}
+
func TestFloat16StructField(t *testing.T) {
type StructWithFloat16 struct {
F16 float16.Float16
diff --git a/go/fory/tests/metastring_resolver_test.go b/go/fory/tests/metastring_resolver_test.go
index fa833b824..b0b820220 100644
--- a/go/fory/tests/metastring_resolver_test.go
+++ b/go/fory/tests/metastring_resolver_test.go
@@ -46,7 +46,7 @@ func TestMetaStringResolver(t *testing.T) {
// Test 2: Manually constructed MetaStringBytes
data2 := []byte{0xBF, 0x05, 0xA4, 0x71, 0xA9, 0x92, 0x53, 0x96, 0xA6, 0x49, 0x4F, 0x72, 0x9C, 0x68, 0x29, 0x80}
- metaBytes2 := fory.NewMetaStringBytes(data2, int64(-5456063526933366015))
+ metaBytes2 := fory.NewMetaStringBytes(data2, fory.ComputeMetaStringHash(data2, meta.LOWER_SPECIAL))
resolver.WriteMetaStringBytes(buffer, metaBytes2, &bufErr)
if bufErr.HasError() {
t.Fatalf("write failed: %v", bufErr.Error())
diff --git a/go/fory/time.go b/go/fory/time.go
index 676efe00e..1a389ccbb 100644
--- a/go/fory/time.go
+++ b/go/fory/time.go
@@ -159,8 +159,8 @@ func (s dateSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(DATE) {
+ return
}
if ctx.HasError() {
return
@@ -268,8 +268,8 @@ func (s durationSerializer) Read(ctx *ReadContext, refMode RefMode, readType boo
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(DURATION) {
+ return
}
if ctx.HasError() {
return
@@ -311,8 +311,8 @@ func (s timeSerializer) Read(ctx *ReadContext, refMode RefMode, readType bool, h
return
}
}
- if readType {
- _ = ctx.buffer.ReadUint8(err)
+ if readType && !ctx.readExpectedTypeID(TIMESTAMP) {
+ return
}
if ctx.HasError() {
return
diff --git a/go/fory/type_def.go b/go/fory/type_def.go
index 81bfa76e8..e1ab7c3eb 100644
--- a/go/fory/type_def.go
+++ b/go/fory/type_def.go
@@ -18,10 +18,7 @@
package fory
import (
- "bytes"
- "compress/zlib"
"fmt"
- "io"
"reflect"
"strings"
@@ -29,17 +26,17 @@ import (
)
const (
- META_SIZE_MASK = 0xFF
- COMPRESS_META_FLAG = 0b1 << 9
- HAS_FIELDS_META_FLAG = 0b1 << 8
- NUM_HASH_BITS = 50
+ META_SIZE_MASK = 0xFF
+ COMPRESS_META_FLAG = 0b1 << 8
+ RESERVED_META_BITS = 0b111 << 9
+ NUM_HASH_BITS = 52
)
/*
TypeDef represents a transportable value object containing type information and field definitions.
typeDef are layout as following:
- - first 8 bytes: global header (50 bits hash + 1 bit compress flag + write fields meta + 8 bits meta size)
- - next 1 byte: meta header (2 bits reserved + 1 bit register by name flag + 5 bits num fields)
+ - first 8 bytes: global header (52 bits hash + 1 bit compress flag + 8 bits meta size)
+ - next 1 byte: kind header
- next variable bytes: type id (varint) or ns name + type name
- next variable bytes: field definitions (see below)
*/
@@ -218,21 +215,21 @@ func (td *TypeDef) buildTypeInfoWithResolver(resolver *TypeResolver) (TypeInfo,
type_ := td.type_
var serializer Serializer
- // For extension types, use the registered serializer if available
- if type_ != nil && resolver != nil {
- if existingSerializer, ok := resolver.typeToSerializers[type_]; ok {
- // Only use registered serializer for extension types (not struct types)
- if _, isExt := existingSerializer.(*extensionSerializerAdapter); isExt {
- serializer = existingSerializer
- } else if ptrSer, isPtrSer := existingSerializer.(*ptrToValueSerializer); isPtrSer {
- if _, isExtInner := ptrSer.valueSerializer.(*extensionSerializerAdapter); isExtInner {
- serializer = existingSerializer
- }
+ if !isStructTypeId(TypeId(td.typeId)) {
+ if type_ != nil && resolver != nil {
+ var err error
+ serializer, err = resolver.getSerializerByType(type_, false)
+ if err != nil {
+ return TypeInfo{}, err
}
}
- }
- // If no extension serializer, create struct serializer
- if serializer == nil {
+ if serializer == nil && resolver != nil {
+ serializer = resolver.getSerializerByTypeID(td.typeId)
+ }
+ if serializer == nil {
+ return TypeInfo{}, fmt.Errorf("no serializer registered for TypeDef kind %d", td.typeId)
+ }
+ } else {
if type_ == nil {
// Unknown struct type - use skipStructSerializer to skip data
serializer = &skipStructSerializer{
@@ -287,6 +284,9 @@ func readTypeDef(fory *Fory, buffer *ByteBuffer, header int64, err *Error) *Type
}
func skipTypeDef(buffer *ByteBuffer, header int64, err *Error) {
+ // Header-cache hits intentionally treat the current body as opaque bytes and skip by the size in
+ // the current header. Parsed TypeDefs are published to the cache only after successful body parse
+ // and 52-bit body-hash validation; cache hits must not reparse or rehash that body.
sz := int(header & META_SIZE_MASK)
if sz == META_SIZE_MASK {
sz += int(buffer.ReadVarUint32(err))
@@ -304,7 +304,14 @@ func readPkgName(buffer *ByteBuffer, namespaceDecoder *meta.Decoder, err *Error)
encodingFlags := header & 0b11 // 2 bits for encoding
size := header >> 2 // 6 bits for size
if size == BIG_NAME_THRESHOLD {
- size = int(buffer.ReadVarUint32Small7(err)) + BIG_NAME_THRESHOLD
+ extra := buffer.ReadVarUint32Small7(err)
+ if err.HasError() {
+ return "", err.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-BIG_NAME_THRESHOLD) {
+ return "", fmt.Errorf("invalid TypeDef namespace length")
+ }
+ size = int(extra) + BIG_NAME_THRESHOLD
}
var encoding meta.Encoding
@@ -319,6 +326,9 @@ func readPkgName(buffer *ByteBuffer, namespaceDecoder *meta.Decoder, err *Error)
return "", fmt.Errorf("invalid package encoding flags: %d", encodingFlags)
}
+ if size > buffer.remaining() {
+ return "", fmt.Errorf("TypeDef namespace length %d exceeds remaining metadata %d", size, buffer.remaining())
+ }
data := make([]byte, size)
if _, err := buffer.Read(data); err != nil {
return "", err
@@ -335,7 +345,14 @@ func readTypeName(buffer *ByteBuffer, typeNameDecoder *meta.Decoder, err *Error)
encodingFlags := header & 0b11 // 2 bits for encoding
size := header >> 2 // 6 bits for size
if size == BIG_NAME_THRESHOLD {
- size = int(buffer.ReadVarUint32Small7(err)) + BIG_NAME_THRESHOLD
+ extra := buffer.ReadVarUint32Small7(err)
+ if err.HasError() {
+ return "", err.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-BIG_NAME_THRESHOLD) {
+ return "", fmt.Errorf("invalid TypeDef typename length")
+ }
+ size = int(extra) + BIG_NAME_THRESHOLD
}
var encoding meta.Encoding
@@ -352,6 +369,9 @@ func readTypeName(buffer *ByteBuffer, typeNameDecoder *meta.Decoder, err *Error)
return "", fmt.Errorf("invalid typename encoding flags: %d", encodingFlags)
}
+ if size > buffer.remaining() {
+ return "", fmt.Errorf("TypeDef typename length %d exceeds remaining metadata %d", size, buffer.remaining())
+ }
data := make([]byte, size)
if _, err := buffer.Read(data); err != nil {
return "", err
@@ -362,16 +382,18 @@ func readTypeName(buffer *ByteBuffer, typeNameDecoder *meta.Decoder, err *Error)
// buildTypeDef constructs a TypeDef from a value
func buildTypeDef(fory *Fory, value reflect.Value) (*TypeDef, error) {
- fieldDefs, err := buildFieldDefs(fory, value)
- if err != nil {
- return nil, fmt.Errorf("failed to extract field infos: %w", err)
- }
-
infoPtr, err := fory.typeResolver.getTypeInfo(value, true)
if err != nil {
return nil, fmt.Errorf("failed to get type info for value %v: %w", value, err)
}
typeId := uint32(infoPtr.TypeID)
+ var fieldDefs []FieldDef
+ if isStructTypeId(TypeId(typeId)) {
+ fieldDefs, err = buildFieldDefs(fory, value)
+ if err != nil {
+ return nil, fmt.Errorf("failed to extract field infos: %w", err)
+ }
+ }
registerByName := IsNamespacedType(TypeId(typeId))
typeDef := NewTypeDef(typeId, infoPtr.UserTypeID, infoPtr.PkgPathBytes, infoPtr.NameBytes, registerByName, false, fieldDefs)
@@ -616,7 +638,9 @@ func newTypeSpecForTypeID(typeID TypeId) (*TypeSpec, error) {
const (
SmallNumFieldsThreshold = 31
- REGISTER_BY_NAME_FLAG = 0b1 << 5
+ RegisterByNameFlag = 0b0010_0000
+ CompatibleTypeDefFlag = 0b0100_0000
+ StructTypeDefFlag = 0b1000_0000
FieldNameSizeThreshold = 15
)
@@ -648,8 +672,8 @@ func getFieldNameEncodingIndex(encoding meta.Encoding) int {
/*
encodingTypeDef encodes a TypeDef into binary format according to the specification
typeDef are layout as following:
-- first 8 bytes: global header (50 bits hash + 1 bit compress flag + write fields meta + 8 bits meta size)
-- next 1 byte: meta header (2 bits reserved + 1 bit register by name flag + 5 bits num fields)
+- first 8 bytes: global header (52 bits hash + 1 bit compress flag + 8 bits meta size)
+- next 1 byte: kind header
- next variable bytes: type id (varint) or ns name + type name
- next variable bytes: field defs (see below)
*/
@@ -758,20 +782,21 @@ func encodingTypeDef(typeResolver *TypeResolver, typeDef *TypeDef) ([]byte, erro
return nil, fmt.Errorf("failed to write typename: %w", err)
}
} else {
- buffer.WriteUint8(uint8(typeDef.typeId))
if typeDef.userTypeId == invalidUserTypeID {
return nil, fmt.Errorf("missing user type ID for typeID %d", typeDef.typeId)
}
buffer.WriteVarUint32(typeDef.userTypeId)
}
- if err := writeFieldDefs(typeResolver, buffer, typeDef.fieldDefs); err != nil {
- return nil, fmt.Errorf("failed to write fields def: %w", err)
+ if isStructTypeId(TypeId(typeDef.typeId)) {
+ if err := writeFieldDefs(typeResolver, buffer, typeDef.fieldDefs); err != nil {
+ return nil, fmt.Errorf("failed to write fields def: %w", err)
+ }
+ } else if len(typeDef.fieldDefs) != 0 {
+ return nil, fmt.Errorf("non-struct TypeDef %d cannot carry field metadata", typeDef.typeId)
}
- // Temporary xlang behavior: keep TypeMeta uncompressed.
- // Some runtimes still do not support TypeMeta decompression.
- result, err := prependGlobalHeader(buffer, false, len(typeDef.fieldDefs) > 0)
+ result, err := prependGlobalHeader(buffer, false)
if err != nil {
return nil, fmt.Errorf("failed to write global binary header: %w", err)
}
@@ -780,16 +805,11 @@ func encodingTypeDef(typeResolver *TypeResolver, typeDef *TypeDef) ([]byte, erro
}
// prependGlobalHeader writes the 8-byte global header
-func prependGlobalHeader(buffer *ByteBuffer, isCompressed bool, hasFieldsMeta bool) (*ByteBuffer, error) {
+func prependGlobalHeader(buffer *ByteBuffer, isCompressed bool) (*ByteBuffer, error) {
var header uint64
metaSize := buffer.WriterIndex()
- hashValue := Murmur3Sum64WithSeed(buffer.GetByteSlice(0, metaSize), 47)
- header |= hashValue << (64 - NUM_HASH_BITS)
-
- if hasFieldsMeta {
- header |= HAS_FIELDS_META_FLAG
- }
+ header |= typeDefHeaderHash(buffer.GetByteSlice(0, metaSize))
if isCompressed {
header |= COMPRESS_META_FLAG
@@ -814,25 +834,82 @@ func prependGlobalHeader(buffer *ByteBuffer, isCompressed bool, hasFieldsMeta bo
// writeMetaHeader writes the 1-byte meta header
func writeMetaHeader(buffer *ByteBuffer, typeDef *TypeDef) error {
- // 2 bits reserved + 1 bit register by name flag + 5 bits num fields
offset := buffer.writerIndex
if err := buffer.WriteByte(0xFF); err != nil {
return err
}
fieldInfos := typeDef.fieldDefs
- header := len(fieldInfos)
- if header > SmallNumFieldsThreshold {
- header = SmallNumFieldsThreshold
- buffer.WriteVarUint32(uint32(len(fieldInfos) - SmallNumFieldsThreshold))
- }
- if typeDef.registerByName {
- header |= REGISTER_BY_NAME_FLAG
+ typeID := TypeId(typeDef.typeId)
+ var header int
+ if isStructTypeId(typeID) {
+ fieldCount := len(fieldInfos)
+ inlineFieldCount := fieldCount
+ if inlineFieldCount > SmallNumFieldsThreshold {
+ inlineFieldCount = SmallNumFieldsThreshold
+ }
+ header = StructTypeDefFlag | inlineFieldCount
+ if typeID == COMPATIBLE_STRUCT || typeID == NAMED_COMPATIBLE_STRUCT {
+ header |= CompatibleTypeDefFlag
+ }
+ if fieldCount >= SmallNumFieldsThreshold {
+ buffer.WriteVarUint32(uint32(fieldCount - SmallNumFieldsThreshold))
+ }
+ if typeDef.registerByName {
+ header |= RegisterByNameFlag
+ }
+ } else {
+ if len(fieldInfos) != 0 {
+ return fmt.Errorf("non-struct TypeDef %d cannot carry field metadata", typeDef.typeId)
+ }
+ kindCode, err := xlangNonStructKindCode(typeID)
+ if err != nil {
+ return err
+ }
+ header = kindCode
}
buffer.PutUint8(offset, uint8(header))
return nil
}
+func xlangNonStructKindCode(typeID TypeId) (int, error) {
+ switch typeID {
+ case ENUM:
+ return 0, nil
+ case NAMED_ENUM:
+ return 1, nil
+ case EXT:
+ return 2, nil
+ case NAMED_EXT:
+ return 3, nil
+ case TYPED_UNION:
+ return 4, nil
+ case NAMED_UNION:
+ return 5, nil
+ default:
+ return 0, fmt.Errorf("unsupported TypeDef kind %d", typeID)
+ }
+}
+
+func xlangNonStructTypeID(kindCode int) (TypeId, error) {
+ switch kindCode {
+ case 0:
+ return ENUM, nil
+ case 1:
+ return NAMED_ENUM, nil
+ case 2:
+ return EXT, nil
+ case 3:
+ return NAMED_EXT, nil
+ case 4:
+ return TYPED_UNION, nil
+ case 5:
+ return NAMED_UNION, nil
+ default:
+ return UNKNOWN, fmt.Errorf("unsupported TypeDef kind code %d", kindCode)
+ }
+}
+
// writeFieldDefs writes field definitions according to the specification
// field def layout as following:
// - first 1 byte: header (2 bits field name encoding + 4 bits size + nullability flag + ref tracking flag)
@@ -913,8 +990,8 @@ func writeFieldDef(typeResolver *TypeResolver, buffer *ByteBuffer, field FieldDe
/*
decodeTypeDef decodes a TypeDef from the buffer
typeDef are layout as following:
- - first 8 bytes: global header (50 bits hash + 1 bit compress flag + write fields meta + 8 bits meta size)
- - next 1 byte: meta header (2 bits reserved + 1 bit register by name flag + 5 bits num fields)
+ - first 8 bytes: global header (52 bits hash + 1 bit compress flag + 8 bits meta size)
+ - next 1 byte: kind header
- next variable bytes: type id (varint) or ns name + type name
- next variable bytes: field definitions (see below)
*/
@@ -922,49 +999,84 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
// ReadData 8-byte global header
var bufErr Error
globalHeader := header
- hasFieldsMeta := (globalHeader & HAS_FIELDS_META_FLAG) != 0
+ if (globalHeader & RESERVED_META_BITS) != 0 {
+ return nil, fmt.Errorf("invalid TypeDef global header")
+ }
isCompressed := (globalHeader & COMPRESS_META_FLAG) != 0
+ if isCompressed {
+ return nil, fmt.Errorf("compressed xlang TypeDef is not supported")
+ }
metaSizeBits := int(globalHeader & META_SIZE_MASK)
metaSize := metaSizeBits
extraMetaSize := 0
if metaSizeBits == META_SIZE_MASK {
- extraMetaSize = int(buffer.ReadVarUint32(&bufErr))
+ extra := buffer.ReadVarUint32(&bufErr)
+ if bufErr.HasError() {
+ return nil, bufErr.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-metaSize) {
+ return nil, fmt.Errorf("invalid TypeDef metadata size")
+ }
+ extraMetaSize = int(extra)
metaSize += extraMetaSize
}
+ if metaSize > fory.config.MaxBinarySize {
+ return nil, MaxBinarySizeExceededError(metaSize, fory.config.MaxBinarySize)
+ }
// Store the encoded bytes for the TypeDef (including meta header and metadata)
encodedMeta := buffer.ReadBinary(metaSize, &bufErr)
if bufErr.HasError() {
return nil, bufErr.TakeError()
}
- decodedMeta := encodedMeta
- if isCompressed {
- decodedMetaBytes, err := decompressMeta(encodedMeta)
- if err != nil {
- return nil, err
- }
- decodedMeta = decodedMetaBytes
- }
- metaBuffer := NewByteBuffer(decodedMeta)
+ metaBuffer := NewByteBuffer(encodedMeta)
var metaErr Error
// ReadData 1-byte meta header
metaHeaderByte := metaBuffer.ReadByte(&metaErr)
- // Extract field count from lower 5 bits
- fieldCount := int(metaHeaderByte & SmallNumFieldsThreshold)
- if fieldCount == SmallNumFieldsThreshold {
- fieldCount += int(metaBuffer.ReadVarUint32(&metaErr))
- }
- if fieldCount > fory.config.MaxTypeFields || fieldCount > metaBuffer.remaining() {
- return nil, fmt.Errorf("field count exceeds maximum allowed limit or available buffer size")
- }
- registeredByName := (metaHeaderByte & REGISTER_BY_NAME_FLAG) != 0
+ isStruct := (metaHeaderByte & StructTypeDefFlag) != 0
+ fieldCount := 0
+ registeredByName := false
// ReadData name or type ID according to the registerByName flag
var typeId uint32
userTypeId := invalidUserTypeID
var nsBytes, nameBytes *MetaStringBytes
var type_ reflect.Type
+ if isStruct {
+ registeredByName = (metaHeaderByte & RegisterByNameFlag) != 0
+ fieldCount = int(metaHeaderByte & SmallNumFieldsThreshold)
+ if fieldCount == SmallNumFieldsThreshold {
+ fieldCount += int(metaBuffer.ReadVarUint32(&metaErr))
+ }
+ if metaErr.HasError() {
+ return nil, metaErr.TakeError()
+ }
+ if fieldCount > fory.config.MaxTypeFields || fieldCount > metaBuffer.remaining() {
+ return nil, fmt.Errorf("field count exceeds maximum allowed limit or available buffer size")
+ }
+ if registeredByName {
+ if (metaHeaderByte & CompatibleTypeDefFlag) != 0 {
+ typeId = uint32(NAMED_COMPATIBLE_STRUCT)
+ } else {
+ typeId = uint32(NAMED_STRUCT)
+ }
+ } else if (metaHeaderByte & CompatibleTypeDefFlag) != 0 {
+ typeId = uint32(COMPATIBLE_STRUCT)
+ } else {
+ typeId = uint32(STRUCT)
+ }
+ } else {
+ if (metaHeaderByte & 0b0111_0000) != 0 {
+ return nil, fmt.Errorf("invalid TypeDef kind header")
+ }
+ kindType, err := xlangNonStructTypeID(int(metaHeaderByte & 0b1111))
+ if err != nil {
+ return nil, err
+ }
+ typeId = uint32(kindType)
+ registeredByName = IsNamespacedType(kindType)
+ }
if registeredByName {
// ReadData namespace and type name for namespaced types
// NOTE: TypeDefs use simple name format, not meta string format with dynamic IDs
@@ -974,7 +1086,14 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
nsEncodingFlags := nsHeader & 0b11 // 2 bits for encoding
nsSize := nsHeader >> 2 // 6 bits for size
if nsSize == BIG_NAME_THRESHOLD {
- nsSize = int(metaBuffer.ReadVarUint32Small7(&metaErr)) + BIG_NAME_THRESHOLD
+ extra := metaBuffer.ReadVarUint32Small7(&metaErr)
+ if metaErr.HasError() {
+ return nil, metaErr.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-BIG_NAME_THRESHOLD) {
+ return nil, fmt.Errorf("invalid TypeDef namespace length")
+ }
+ nsSize = int(extra) + BIG_NAME_THRESHOLD
}
// Java pkg encoding: 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL
@@ -989,6 +1108,9 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
default:
return nil, fmt.Errorf("invalid package encoding flags: %d", nsEncodingFlags)
}
+ if nsSize > metaBuffer.remaining() {
+ return nil, fmt.Errorf("TypeDef namespace length %d exceeds remaining metadata %d", nsSize, metaBuffer.remaining())
+ }
nsData := make([]byte, nsSize)
if _, err := metaBuffer.Read(nsData); err != nil {
return nil, fmt.Errorf("failed to read namespace data: %w", err)
@@ -1000,7 +1122,14 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
typeEncodingFlags := typeHeader & 0b11 // 2 bits for encoding
typeSize := typeHeader >> 2 // 6 bits for size
if typeSize == BIG_NAME_THRESHOLD {
- typeSize = int(metaBuffer.ReadVarUint32Small7(&metaErr)) + BIG_NAME_THRESHOLD
+ extra := metaBuffer.ReadVarUint32Small7(&metaErr)
+ if metaErr.HasError() {
+ return nil, metaErr.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-BIG_NAME_THRESHOLD) {
+ return nil, fmt.Errorf("invalid TypeDef typename length")
+ }
+ typeSize = int(extra) + BIG_NAME_THRESHOLD
}
// Java typename encoding: 0=UTF_8, 1=ALL_TO_LOWER_SPECIAL, 2=LOWER_UPPER_DIGIT_SPECIAL, 3=FIRST_TO_LOWER_SPECIAL
@@ -1017,6 +1146,9 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
default:
return nil, fmt.Errorf("invalid typename encoding flags: %d", typeEncodingFlags)
}
+ if typeSize > metaBuffer.remaining() {
+ return nil, fmt.Errorf("TypeDef typename length %d exceeds remaining metadata %d", typeSize, metaBuffer.remaining())
+ }
typeData := make([]byte, typeSize)
if _, err := metaBuffer.Read(typeData); err != nil {
return nil, fmt.Errorf("failed to read typename data: %w", err)
@@ -1049,7 +1181,9 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
if fallbackInfo, fallbackExists := fory.typeResolver.namedTypeToTypeInfo[nameKey]; fallbackExists {
info = fallbackInfo
exists = true
- fory.typeResolver.nsTypeToTypeInfo[nsTypeKey{nsBytes.Hashcode, nameBytes.Hashcode}] = info
+ if len(fory.typeResolver.nsTypeToTypeInfo) < maxCachedNamedTypeInfos {
+ fory.typeResolver.nsTypeToTypeInfo[nsTypeKey{nsBytes.Hashcode, nameBytes.Hashcode}] = info
+ }
}
}
if exists {
@@ -1059,18 +1193,19 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
if type_.Kind() == reflect.Ptr {
type_ = type_.Elem()
}
- typeId = uint32(info.TypeID)
+ if uint32(info.TypeID) != typeId {
+ return nil, fmt.Errorf("TypeDef kind does not match registered type metadata")
+ }
userTypeId = info.UserTypeID
} else {
- // Type not registered - use NAMED_STRUCT as default typeId
- // The type_ will remain nil and will be set from field definitions later
- typeId = uint32(NAMED_STRUCT)
type_ = nil
}
} else {
- typeId = uint32(metaBuffer.ReadUint8(&metaErr))
userTypeId = metaBuffer.ReadVarUint32(&metaErr)
if info, exists := fory.typeResolver.userTypeIdToTypeInfo[userTypeId]; exists {
+ if uint32(info.TypeID) != typeId {
+ return nil, fmt.Errorf("TypeDef kind does not match registered type metadata")
+ }
type_ = info.Type
} else if info, exists := fory.typeResolver.typeIDToTypeInfo[typeId]; exists {
type_ = info.Type
@@ -1083,14 +1218,24 @@ func decodeTypeDef(fory *Fory, buffer *ByteBuffer, header int64) (*TypeDef, erro
// ReadData fields information
fieldInfos := make([]FieldDef, fieldCount)
- if hasFieldsMeta {
- for i := 0; i < fieldCount; i++ {
- fieldInfo, err := readFieldDef(fory.typeResolver, metaBuffer)
- if err != nil {
- return nil, fmt.Errorf("failed to read field def %d: %w", i, err)
- }
- fieldInfos[i] = fieldInfo
+ for i := 0; i < fieldCount; i++ {
+ fieldInfo, err := readFieldDef(fory.typeResolver, metaBuffer)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read field def %d: %w", i, err)
}
+ fieldInfos[i] = fieldInfo
+ }
+ if !isStruct && len(fieldInfos) != 0 {
+ return nil, fmt.Errorf("non-struct TypeDef cannot carry field metadata")
+ }
+ if metaErr.HasError() {
+ return nil, metaErr.TakeError()
+ }
+ if remaining := metaBuffer.remaining(); remaining != 0 {
+ return nil, fmt.Errorf("TypeDef metadata body has %d trailing bytes", remaining)
+ }
+ if err := validateParsedTypeDefHash(globalHeader, metaSizeBits, extraMetaSize, encodedMeta); err != nil {
+ return nil, err
}
encoded := buildTypeDefEncoded(globalHeader, metaSizeBits, extraMetaSize, encodedMeta)
@@ -1130,17 +1275,32 @@ func buildTypeDefEncoded(header int64, metaSizeBits, extraMetaSize int, metaByte
return buffer.Bytes()
}
-func decompressMeta(encoded []byte) ([]byte, error) {
- reader, err := zlib.NewReader(bytes.NewReader(encoded))
- if err != nil {
- return nil, fmt.Errorf("failed to create meta decompressor: %w", err)
+func typeDefHeaderHash(data []byte) uint64 {
+ hash := int64(Murmur3Sum64WithSeed(data, 47) << (64 - NUM_HASH_BITS))
+ if hash < 0 {
+ hash = -hash
}
- defer reader.Close()
- decoded, err := io.ReadAll(reader)
- if err != nil {
- return nil, fmt.Errorf("failed to decompress meta: %w", err)
+ hashMask := ^uint64(0)
+ hashMask <<= uint(64 - NUM_HASH_BITS)
+ return uint64(hash) & hashMask
+}
+
+func validateParsedTypeDefHash(header int64, metaSizeBits, extraMetaSize int, encoded []byte) error {
+ size := metaSizeBits
+ if size == META_SIZE_MASK {
+ size += extraMetaSize
}
- return decoded, nil
+ if len(encoded) != size {
+ return fmt.Errorf("invalid TypeDef encoded size")
+ }
+ hashMask := ^uint64(0)
+ hashMask <<= uint(64 - NUM_HASH_BITS)
+ expectedHeaderHash := typeDefHeaderHash(encoded)
+ actualHeaderHash := uint64(header) & hashMask
+ if expectedHeaderHash != actualHeaderHash {
+ return fmt.Errorf("invalid TypeDef metadata hash")
+ }
+ return nil
}
/*
@@ -1174,7 +1334,14 @@ func readFieldDef(typeResolver *TypeResolver, buffer *ByteBuffer) (FieldDef, err
// Read tag ID
tagID := sizeBits
if sizeBits == 0x0F {
- tagID = FieldNameSizeThreshold + int(buffer.ReadVarUint32(&bufErr))
+ extra := buffer.ReadVarUint32(&bufErr)
+ if bufErr.HasError() {
+ return FieldDef{}, bufErr.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-FieldNameSizeThreshold) {
+ return FieldDef{}, fmt.Errorf("invalid TypeDef field tag ID")
+ }
+ tagID = FieldNameSizeThreshold + int(extra)
}
// Read field type
@@ -1182,6 +1349,9 @@ func readFieldDef(typeResolver *TypeResolver, buffer *ByteBuffer) (FieldDef, err
if err != nil {
return FieldDef{}, err
}
+ if bufErr.HasError() {
+ return FieldDef{}, bufErr.TakeError()
+ }
return FieldDef{
name: "", // No field name when using tag ID
@@ -1197,7 +1367,14 @@ func readFieldDef(typeResolver *TypeResolver, buffer *ByteBuffer) (FieldDef, err
nameEncoding := fieldNameEncodings[nameEncodingFlag]
nameLen := sizeBits
if nameLen == 0x0F {
- nameLen = FieldNameSizeThreshold + int(buffer.ReadVarUint32(&bufErr))
+ extra := buffer.ReadVarUint32(&bufErr)
+ if bufErr.HasError() {
+ return FieldDef{}, bufErr.TakeError()
+ }
+ if uint64(extra) > uint64(MaxInt-FieldNameSizeThreshold) {
+ return FieldDef{}, fmt.Errorf("invalid TypeDef field name length")
+ }
+ nameLen = FieldNameSizeThreshold + int(extra)
} else {
nameLen++ // Adjust for 1-based encoding
}
@@ -1207,9 +1384,18 @@ func readFieldDef(typeResolver *TypeResolver, buffer *ByteBuffer) (FieldDef, err
if err != nil {
return FieldDef{}, err
}
+ if bufErr.HasError() {
+ return FieldDef{}, bufErr.TakeError()
+ }
// Read field name based on encoding
+ if nameLen > buffer.remaining() {
+ return FieldDef{}, fmt.Errorf("TypeDef field name length %d exceeds remaining metadata %d", nameLen, buffer.remaining())
+ }
nameBytes := buffer.ReadBinary(nameLen, &bufErr)
+ if bufErr.HasError() {
+ return FieldDef{}, bufErr.TakeError()
+ }
fieldName, err := typeResolver.typeNameDecoder.Decode(nameBytes, nameEncoding)
if err != nil {
return FieldDef{}, fmt.Errorf("failed to decode field name: %w", err)
diff --git a/go/fory/type_def_test.go b/go/fory/type_def_test.go
index 5e88bd174..37b465001 100644
--- a/go/fory/type_def_test.go
+++ b/go/fory/type_def_test.go
@@ -18,10 +18,13 @@
package fory
import (
+ "bytes"
+ "compress/zlib"
"reflect"
"testing"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
// Test structs for encoding/decoding
@@ -176,6 +179,35 @@ func checkTypeSpecRecursivelyOrNil(t *testing.T, original, decoded *TypeSpec, pa
checkTypeSpecRecursively(t, original, decoded, path, compareRootFlags)
}
+func typeDefTestBodyWithoutFields() []byte {
+ buffer := NewByteBuffer(nil)
+ buffer.WriteByte(StructTypeDefFlag)
+ buffer.WriteVarUint32(0)
+ return buffer.Bytes()
+}
+
+func typeDefTestFrame(t *testing.T, body []byte, compressed bool) (*ByteBuffer, int64) {
+ t.Helper()
+ bodyBuffer := NewByteBuffer(nil)
+ bodyBuffer.WriteBinary(body)
+ frame, err := prependGlobalHeader(bodyBuffer, compressed)
+ require.NoError(t, err)
+ readErr := &Error{}
+ header := frame.ReadInt64(readErr)
+ require.NoError(t, readErr.CheckError())
+ return frame, header
+}
+
+func deflateTypeDefTestBody(t *testing.T, body []byte) []byte {
+ t.Helper()
+ var compressed bytes.Buffer
+ writer := zlib.NewWriter(&compressed)
+ _, err := writer.Write(body)
+ require.NoError(t, err)
+ require.NoError(t, writer.Close())
+ return compressed.Bytes()
+}
+
// Item1 struct with mixed nullable (pointer) and non-nullable (primitive) fields
type Item1 struct {
F1 int32
@@ -301,22 +333,196 @@ func TestTypeDefNullableFields(t *testing.T) {
// allocation that would OOM-crash the process.
func TestTypeDefFieldCountOOMPanic(t *testing.T) {
fory := NewFory()
- header := int64(HAS_FIELDS_META_FLAG | 8)
- // metaHeaderByte value of 31 triggers the extended VarUint32 field-count path.
buffer := NewByteBuffer(make([]byte, 0, 8))
- buffer.WriteByte(31)
+ buffer.WriteByte(StructTypeDefFlag | SmallNumFieldsThreshold)
buffer.WriteVarUint32(2000000000)
- buffer.WriteUint8(0)
- buffer.WriteVarUint32(0)
buffer.SetReaderIndex(0)
- _, err := decodeTypeDef(fory, buffer, header)
+ _, err := decodeTypeDef(fory, buffer, int64(buffer.WriterIndex()))
if err == nil {
t.Fatal("expected error for oversized fieldCount, got nil")
}
}
+func TestTypeDefRejectsReservedGlobalHeaderBits(t *testing.T) {
+ fory := NewFory()
+ buffer := NewByteBuffer(nil)
+ buffer.WriteByte(StructTypeDefFlag)
+ buffer.WriteVarUint32(0)
+ buffer.SetReaderIndex(0)
+
+ _, err := decodeTypeDef(fory, buffer, int64(RESERVED_META_BITS|uint64(buffer.WriterIndex())))
+ if err == nil {
+ t.Fatal("expected error for reserved TypeDef global header bits")
+ }
+}
+
+func TestTypeDefRejectsReservedNonStructKindBits(t *testing.T) {
+ fory := NewFory()
+ body := []byte{0b0001_0000}
+ frame, header := typeDefTestFrame(t, body, false)
+
+ _, err := decodeTypeDef(fory, frame, header)
+ if err == nil {
+ t.Fatal("expected error for reserved non-struct TypeDef kind bits")
+ }
+}
+
+func TestTypeDefRejectsTrailingMetadataBytes(t *testing.T) {
+ fory := NewFory()
+ meta := NewByteBuffer(nil)
+ meta.WriteByte(StructTypeDefFlag)
+ meta.WriteVarUint32(0)
+ meta.WriteByte(0xff)
+
+ buffer := NewByteBuffer(nil)
+ _, writeErr := buffer.Write(meta.Bytes())
+ if writeErr != nil {
+ t.Fatalf("failed to write type def metadata: %v", writeErr)
+ }
+ buffer.SetReaderIndex(0)
+
+ _, err := decodeTypeDef(fory, buffer, int64(len(meta.Bytes())))
+ if err == nil {
+ t.Fatal("expected error for trailing TypeDef metadata bytes")
+ }
+}
+
+func TestTypeDefExtendedFieldCountHeaderDoesNotSetRegisterByName(t *testing.T) {
+ fields := make([]FieldDef, 32)
+ for i := range fields {
+ fields[i] = FieldDef{
+ typeSpec: NewSimpleTypeSpec(INT32),
+ tagID: i + 1,
+ }
+ }
+ typeDef := NewTypeDef(uint32(STRUCT), 700, nil, nil, false, false, fields)
+ buffer := NewByteBuffer(nil)
+
+ require.NoError(t, writeMetaHeader(buffer, typeDef))
+ header := buffer.Bytes()[0]
+ require.Equal(t, byte(StructTypeDefFlag|SmallNumFieldsThreshold), header)
+ require.Zero(t, header&RegisterByNameFlag)
+}
+
+func TestTypeDefRejectsMetadataHashMismatch(t *testing.T) {
+ fory := NewFory()
+ body := typeDefTestBodyWithoutFields()
+ buffer := NewByteBuffer(nil)
+ buffer.WriteBinary(body)
+ buffer.SetReaderIndex(0)
+
+ _, err := decodeTypeDef(fory, buffer, int64(len(body)))
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "metadata hash")
+}
+
+func TestTypeDefRejectsEncodedMetadataAboveMaxBinarySize(t *testing.T) {
+ fory := NewFory(WithMaxBinarySize(1))
+ body := typeDefTestBodyWithoutFields()
+ frame, header := typeDefTestFrame(t, body, false)
+
+ _, err := decodeTypeDef(fory, frame, header)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "max binary size exceeded")
+}
+
+func TestTypeDefRejectsCompressedMetadata(t *testing.T) {
+ decoded := typeDefTestBodyWithoutFields()
+ compressed := deflateTypeDefTestBody(t, decoded)
+ fory := NewFory(WithMaxBinarySize(4096))
+ frame, header := typeDefTestFrame(t, compressed, true)
+
+ _, err := decodeTypeDef(fory, frame, header)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "compressed xlang TypeDef")
+}
+
+func TestReadSharedTypeMetaCapsParsedTypeDefCache(t *testing.T) {
+ fory := NewFory(WithCompatible(true))
+ require.NoError(t, fory.RegisterNamedStruct(SimpleStruct{}, "example.SimpleStruct"))
+ typeDef, err := buildTypeDef(fory, reflect.ValueOf(SimpleStruct{}))
+ require.NoError(t, err)
+ require.NotEmpty(t, typeDef.encoded)
+
+ for i := 0; i < maxCachedTypeDefs; i++ {
+ fory.typeResolver.defIdToTypeDef[int64(i)] = typeDef
+ }
+ headerErr := &Error{}
+ header := NewByteBuffer(typeDef.encoded).ReadInt64(headerErr)
+ require.NoError(t, headerErr.CheckError())
+ require.NotContains(t, fory.typeResolver.defIdToTypeDef, header)
+
+ buffer := NewByteBuffer(nil)
+ buffer.WriteVarUint32(0)
+ buffer.WriteBinary(typeDef.encoded)
+ readErr := &Error{}
+ typeInfo := fory.typeResolver.readSharedTypeMeta(buffer, readErr)
+ require.NoError(t, readErr.CheckError())
+ require.NotNil(t, typeInfo)
+ require.Len(t, fory.typeResolver.defIdToTypeDef, maxCachedTypeDefs)
+ require.NotContains(t, fory.typeResolver.defIdToTypeDef, header)
+}
+
+func TestDecodeTypeDefFallbackNamedTypeCacheRespectsCap(t *testing.T) {
+ fory := NewFory(WithCompatible(true))
+ require.NoError(t, fory.RegisterNamedStruct(SimpleStruct{}, "example.SimpleStruct"))
+ typeDef, err := buildTypeDef(fory, reflect.ValueOf(SimpleStruct{}))
+ require.NoError(t, err)
+ require.NotNil(t, typeDef.nsName)
+ require.NotNil(t, typeDef.typeName)
+
+ nameKey := nsTypeKey{typeDef.nsName.Hashcode, typeDef.typeName.Hashcode}
+ delete(fory.typeResolver.nsTypeToTypeInfo, nameKey)
+ info := fory.typeResolver.namedTypeToTypeInfo[[2]string{"example", "SimpleStruct"}]
+ require.NotNil(t, info)
+ for i := 0; len(fory.typeResolver.nsTypeToTypeInfo) < maxCachedNamedTypeInfos; i++ {
+ fory.typeResolver.nsTypeToTypeInfo[nsTypeKey{int64(i + 1), int64(i + 2)}] = info
+ }
+ require.NotContains(t, fory.typeResolver.nsTypeToTypeInfo, nameKey)
+
+ buffer := NewByteBuffer(nil)
+ readErr := &Error{}
+ typeDef.writeTypeDef(buffer, readErr)
+ require.NoError(t, readErr.CheckError())
+ header := buffer.ReadInt64(readErr)
+ require.NoError(t, readErr.CheckError())
+ decoded := readTypeDef(fory, buffer, header, readErr)
+ require.NoError(t, readErr.CheckError())
+ require.NotNil(t, decoded)
+ require.Len(t, fory.typeResolver.nsTypeToTypeInfo, maxCachedNamedTypeInfos)
+ require.NotContains(t, fory.typeResolver.nsTypeToTypeInfo, nameKey)
+}
+
+func TestTypeDefRejectsNamespaceLengthBeyondMetadata(t *testing.T) {
+ fory := NewFory()
+ meta := NewByteBuffer(nil)
+ meta.WriteByte(StructTypeDefFlag | RegisterByNameFlag)
+ meta.WriteByte(byte(BIG_NAME_THRESHOLD << 2))
+ meta.WriteVarUint32Small7(100)
+ frame, header := typeDefTestFrame(t, meta.Bytes(), false)
+
+ _, err := decodeTypeDef(fory, frame, header)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "namespace length")
+}
+
+func TestTypeDefRejectsFieldNameLengthBeyondMetadata(t *testing.T) {
+ fory := NewFory()
+ meta := NewByteBuffer(nil)
+ meta.WriteByte(StructTypeDefFlag | 1)
+ meta.WriteVarUint32(0)
+ meta.WriteByte(0x0F << 2)
+ meta.WriteVarUint32(100)
+ meta.WriteUint8(uint8(INT32))
+ frame, header := typeDefTestFrame(t, meta.Bytes(), false)
+
+ _, err := decodeTypeDef(fory, frame, header)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "field name length")
+}
+
// TestTypeDefNestedRecursionStackOverflowPanic verifies that readFieldTypeWithFlags
// rejects a crafted payload with 20 million nested LIST types, returning an error
// at depth 64 instead of recursing until a goroutine stack overflow crashes the process.
diff --git a/go/fory/type_resolver.go b/go/fory/type_resolver.go
index 3f0e7ee84..3838e67e2 100644
--- a/go/fory/type_resolver.go
+++ b/go/fory/type_resolver.go
@@ -52,9 +52,11 @@ const (
useStringId = 1
SMALL_STRING_THRESHOLD = 16
// 0xffffffff is reserved for "unset".
- maxUserTypeID uint32 = 0xfffffffe
- invalidUserTypeID uint32 = 0xffffffff
- internalTypeIDLimit = 0xFF
+ maxUserTypeID uint32 = 0xfffffffe
+ invalidUserTypeID uint32 = 0xffffffff
+ internalTypeIDLimit = 0xFF
+ maxCachedTypeDefs = 8192
+ maxCachedNamedTypeInfos = 8192
)
var (
@@ -1633,6 +1635,8 @@ func (r *TypeResolver) readSharedTypeMeta(buffer *ByteBuffer, err *Error) *TypeI
var td *TypeDef
if existingTd, exists := r.defIdToTypeDef[id]; exists {
+ // Header-cache hits intentionally skip without rehashing. Entries reach this cache only
+ // after a successful TypeDef parse and 52-bit body-hash validation.
skipTypeDef(buffer, id, err)
td = existingTd
} else {
@@ -1640,7 +1644,6 @@ func (r *TypeResolver) readSharedTypeMeta(buffer *ByteBuffer, err *Error) *TypeI
if err.HasError() {
return nil
}
- r.defIdToTypeDef[id] = newTd
td = newTd
}
@@ -1649,6 +1652,9 @@ func (r *TypeResolver) readSharedTypeMeta(buffer *ByteBuffer, err *Error) *TypeI
err.SetError(typeInfoErr)
return nil
}
+ if _, exists := r.defIdToTypeDef[id]; !exists && len(r.defIdToTypeDef) < maxCachedTypeDefs {
+ r.defIdToTypeDef[id] = td
+ }
context.readTypeInfos = append(context.readTypeInfos, typeInfo)
return typeInfo
@@ -2162,7 +2168,9 @@ func (r *TypeResolver) resolveTypeInfoByMetaBytes(nsBytes, typeBytes *MetaString
nameKey := [2]string{ns, typeName}
if typeInfo, exists := r.namedTypeToTypeInfo[nameKey]; exists {
- r.nsTypeToTypeInfo[compositeKey] = typeInfo
+ if len(r.nsTypeToTypeInfo) < maxCachedNamedTypeInfos {
+ r.nsTypeToTypeInfo[compositeKey] = typeInfo
+ }
return typeInfo
}
diff --git a/go/fory/writer.go b/go/fory/writer.go
index 4fccf3711..41ac50300 100644
--- a/go/fory/writer.go
+++ b/go/fory/writer.go
@@ -38,7 +38,7 @@ type WriteContext struct {
depth int
maxDepth int
typeResolver *TypeResolver // For complex type serialization
- refResolver *RefResolver // For reference tracking (legacy)
+ refResolver *RefResolver // For reference tracking in native-mode paths
bufferCallback func(BufferObject) bool // Callback for out-of-band buffers
outOfBand bool // Whether out-of-band serialization is enabled
err Error // Accumulated error state for deferred checking
@@ -108,7 +108,7 @@ func (c *WriteContext) TypeResolver() *TypeResolver {
return c.typeResolver
}
-// RefResolver returns the reference resolver (legacy)
+// RefResolver returns the reference resolver.
func (c *WriteContext) RefResolver() *RefResolver {
return c.refResolver
}
diff --git a/integration_tests/idl_tests/javascript/roundtrip.ts b/integration_tests/idl_tests/javascript/roundtrip.ts
index 23af999e0..4b77e8fc4 100644
--- a/integration_tests/idl_tests/javascript/roundtrip.ts
+++ b/integration_tests/idl_tests/javascript/roundtrip.ts
@@ -123,8 +123,10 @@ function resolveRootSerializer(fory: Fory, bytes: Uint8Array): Serializer {
fory.readContext.reset(bytes);
const reader = fory.readContext.reader;
const bitmap = reader.readUint8();
- if ((bitmap & ConfigFlags.isNullFlag) === ConfigFlags.isNullFlag) {
- throw new Error("IDL roundtrip does not support null root payloads");
+ const supportedBitmap =
+ ConfigFlags.isCrossLanguageFlag | ConfigFlags.isOutOfBandFlag;
+ if ((bitmap & ~supportedBitmap) !== 0) {
+ throw new Error("unsupported root header bitmap");
}
if (
(bitmap & ConfigFlags.isCrossLanguageFlag) !==
diff --git a/java/fory-core/src/main/java/org/apache/fory/Fory.java b/java/fory-core/src/main/java/org/apache/fory/Fory.java
index adf1e8c6d..56f6addbd 100644
--- a/java/fory-core/src/main/java/org/apache/fory/Fory.java
+++ b/java/fory-core/src/main/java/org/apache/fory/Fory.java
@@ -68,7 +68,7 @@ import org.apache.fory.util.StringUtils;
/**
* Cross-language header layout: 1-byte bitmap.
*
- * <p>Bit 0: null flag, Bit 1: xlang flag, Bit 2: out-of-band flag, Bits 3-7 reserved.
+ * <p>Bit 0: xlang flag, Bit 1: out-of-band flag, Bits 2-7 reserved.
*
* <p>serialize/deserialize are the root object APIs. Nested serialization and deserialization go
* through {@link WriteContext} and {@link ReadContext}.
@@ -86,9 +86,9 @@ public final class Fory implements BaseFory {
// this flag indicates that the object is a referencable and first write.
public static final byte REF_VALUE_FLAG = 0;
public static final byte NOT_SUPPORT_XLANG = 0;
- private static final byte isNilFlag = 1;
- private static final byte isCrossLanguageFlag = 1 << 1;
- private static final byte isOutOfBandFlag = 1 << 2;
+ private static final byte isCrossLanguageFlag = 1;
+ private static final byte isOutOfBandFlag = 1 << 1;
+ private static final byte reservedBitmapFlags = (byte) ~0b11;
private final Config config;
private final TypeResolver typeResolver;
@@ -98,6 +98,7 @@ public final class Fory implements BaseFory {
private final WriteContext writeContext;
private final ReadContext readContext;
private final CopyContext copyContext;
+ private final byte headerBitmap;
private MemoryBuffer buffer;
public Fory(ForyBuilder builder, ClassLoader classLoader) {
@@ -119,6 +120,7 @@ public final class Fory implements BaseFory {
this.sharedRegistry = sharedRegistry;
this.classLoader = classLoader;
config = new Config(builder);
+ headerBitmap = config.isXlang() ? isCrossLanguageFlag : 0;
RefWriter refWriter;
RefReader refReader;
if (config.trackingRef()) {
@@ -293,15 +295,7 @@ public final class Fory implements BaseFory {
ensureRegistrationFinished();
writeContext.prepare(buffer, callback);
try {
- byte bitmap = 0;
- if (config.isXlang()) {
- bitmap |= isCrossLanguageFlag;
- }
- if (obj == null) {
- bitmap |= isNilFlag;
- buffer.writeByte(bitmap);
- return buffer;
- }
+ byte bitmap = headerBitmap;
if (callback != null) {
bitmap |= isOutOfBandFlag;
}
@@ -379,12 +373,9 @@ public final class Fory implements BaseFory {
public <T> T deserialize(MemoryBuffer buffer, Class<T> type) {
ensureRegistrationFinished();
byte bitmap = buffer.readByte();
- if ((bitmap & isNilFlag) == isNilFlag) {
- return null;
+ if (bitmap != headerBitmap) {
+ checkHeaderBitmapWithoutOutOfBand(bitmap);
}
- boolean peerOutOfBandEnabled = (bitmap & isOutOfBandFlag) == isOutOfBandFlag;
- assert !peerOutOfBandEnabled : "Out of band buffers not passed in when deserializing";
- checkXlangBitmap(bitmap);
readContext.prepare(buffer, null, false);
try {
try {
@@ -449,11 +440,10 @@ public final class Fory implements BaseFory {
public Object deserialize(MemoryBuffer buffer, Iterable<MemoryBuffer> outOfBandBuffers) {
ensureRegistrationFinished();
byte bitmap = buffer.readByte();
- if ((bitmap & isNilFlag) == isNilFlag) {
- return null;
+ boolean peerOutOfBandEnabled = false;
+ if (bitmap != headerBitmap) {
+ peerOutOfBandEnabled = checkHeaderBitmap(bitmap);
}
- checkXlangBitmap(bitmap);
- boolean peerOutOfBandEnabled = (bitmap & isOutOfBandFlag) == isOutOfBandFlag;
if (peerOutOfBandEnabled) {
Preconditions.checkNotNull(
outOfBandBuffers,
@@ -530,13 +520,24 @@ public final class Fory implements BaseFory {
}
}
- private void checkXlangBitmap(byte bitmap) {
+ private void checkHeaderBitmapWithoutOutOfBand(byte bitmap) {
+ if (checkHeaderBitmap(bitmap)) {
+ throw new IllegalArgumentException("Out of band buffers not passed in when deserializing");
+ }
+ }
+
+ private boolean checkHeaderBitmap(byte bitmap) {
+ Preconditions.checkArgument(
+ (bitmap & reservedBitmapFlags) == 0,
+ "Serialized payload uses reserved header bitmap flags 0x%s",
+ Integer.toHexString(Byte.toUnsignedInt((byte) (bitmap & reservedBitmapFlags))));
boolean payloadCrossLanguage = (bitmap & isCrossLanguageFlag) == isCrossLanguageFlag;
Preconditions.checkArgument(
payloadCrossLanguage == config.isXlang(),
"Serialized payload xlang flag %s does not match this Fory mode %s",
payloadCrossLanguage,
config.isXlang());
+ return (bitmap & isOutOfBandFlag) == isOutOfBandFlag;
}
@Override
diff --git a/java/fory-core/src/main/java/org/apache/fory/collection/LongLongByteMap.java b/java/fory-core/src/main/java/org/apache/fory/collection/LongLongByteMap.java
index 3df143179..26bc53190 100644
--- a/java/fory-core/src/main/java/org/apache/fory/collection/LongLongByteMap.java
+++ b/java/fory-core/src/main/java/org/apache/fory/collection/LongLongByteMap.java
@@ -19,6 +19,7 @@
package org.apache.fory.collection;
+import java.util.Arrays;
import org.apache.fory.annotation.Internal;
import org.apache.fory.util.Preconditions;
@@ -129,6 +130,25 @@ public final class LongLongByteMap<V> {
}
}
+ public void clear() {
+ if (size == 0) {
+ return;
+ }
+ size = 0;
+ Arrays.fill(keyTable, null);
+ ObjectArray.clearObjectArray(valueTable, 0, valueTable.length);
+ }
+
+ public void clear(int maximumCapacity) {
+ int tableSize = ForyObjectMap.tableSize(maximumCapacity, loadFactor);
+ if (keyTable.length <= tableSize) {
+ clear();
+ return;
+ }
+ size = 0;
+ resize(tableSize);
+ }
+
private void resize(int newSize) {
int oldCapacity = keyTable.length;
threshold = (int) (newSize * loadFactor);
diff --git a/java/fory-core/src/main/java/org/apache/fory/config/Config.java b/java/fory-core/src/main/java/org/apache/fory/config/Config.java
index c820986e4..bd7c3c710 100644
--- a/java/fory-core/src/main/java/org/apache/fory/config/Config.java
+++ b/java/fory-core/src/main/java/org/apache/fory/config/Config.java
@@ -65,6 +65,8 @@ public class Config implements Serializable {
private final boolean serializeEnumByName;
private final int bufferSizeLimitBytes;
private final int maxDepth;
+ private final int maxBinarySize;
+ private final int maxCollectionSize;
private final float mapRefLoadFactor;
private final boolean forVirtualThread;
@@ -107,6 +109,8 @@ public class Config implements Serializable {
serializeEnumByName = builder.serializeEnumByName;
bufferSizeLimitBytes = builder.bufferSizeLimitBytes;
maxDepth = builder.maxDepth;
+ maxBinarySize = builder.maxBinarySize;
+ maxCollectionSize = builder.maxCollectionSize;
mapRefLoadFactor = builder.mapRefLoadFactor;
forVirtualThread = builder.forVirtualThread;
}
@@ -298,6 +302,16 @@ public class Config implements Serializable {
return maxDepth;
}
+ /** Returns max binary payload size for attacker-controlled binary and primitive-array lengths. */
+ public int maxBinarySize() {
+ return maxBinarySize;
+ }
+
+ /** Returns max collection allocation size for attacker-controlled collection lengths. */
+ public int maxCollectionSize() {
+ return maxCollectionSize;
+ }
+
/** Returns loadFactor of MacRef's writtenObjects. */
public float mapRefLoadFactor() {
return mapRefLoadFactor;
@@ -332,6 +346,8 @@ public class Config implements Serializable {
&& compressIntArray == config.compressIntArray
&& compressLongArray == config.compressLongArray
&& bufferSizeLimitBytes == config.bufferSizeLimitBytes
+ && maxBinarySize == config.maxBinarySize
+ && maxCollectionSize == config.maxCollectionSize
&& requireClassRegistration == config.requireClassRegistration
&& suppressClassRegistrationWarnings == config.suppressClassRegistrationWarnings
&& registerGuavaTypes == config.registerGuavaTypes
@@ -371,6 +387,8 @@ public class Config implements Serializable {
compressIntArray,
compressLongArray,
bufferSizeLimitBytes,
+ maxBinarySize,
+ maxCollectionSize,
requireClassRegistration,
suppressClassRegistrationWarnings,
registerGuavaTypes,
diff --git a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
index 5c3d6a45d..52bd6b08b 100644
--- a/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/config/ForyBuilder.java
@@ -96,6 +96,8 @@ public final class ForyBuilder {
Integer bufferSizeLimitBytes = -1;
MetaCompressor metaCompressor = new DeflaterMetaCompressor();
int maxDepth = 50;
+ int maxBinarySize = 64 * 1024 * 1024;
+ int maxCollectionSize = 1_000_000;
float mapRefLoadFactor = 0.51f;
boolean forVirtualThread = false;
TypeChecker typeChecker;
@@ -473,6 +475,30 @@ public final class ForyBuilder {
return this;
}
+ /**
+ * Set max binary payload size for deserialization. Binary and primitive-array byte lengths above
+ * this limit are rejected before allocation. Default max binary size is 64 MiB.
+ */
+ public ForyBuilder withMaxBinarySize(int maxBinarySize) {
+ Preconditions.checkArgument(
+ maxBinarySize >= 0, "maxBinarySize must >= 0 but got %s", maxBinarySize);
+ this.maxBinarySize = maxBinarySize;
+ recordAction(b -> b.withMaxBinarySize(maxBinarySize));
+ return this;
+ }
+
+ /**
+ * Set max collection size for deserialization. Collection lengths and collection capacity fields
+ * above this limit are rejected before allocation. Default max collection size is 1,000,000.
+ */
+ public ForyBuilder withMaxCollectionSize(int maxCollectionSize) {
+ Preconditions.checkArgument(
+ maxCollectionSize >= 0, "maxCollectionSize must >= 0 but got %s", maxCollectionSize);
+ this.maxCollectionSize = maxCollectionSize;
+ recordAction(b -> b.withMaxCollectionSize(maxCollectionSize));
+ return this;
+ }
+
/** Set loadFactor of MapRefResolver writtenObjects. Default value is 0.51 */
public ForyBuilder withMapRefLoadFactor(float loadFactor) {
Preconditions.checkArgument(
diff --git a/java/fory-core/src/main/java/org/apache/fory/context/MetaStringReader.java b/java/fory-core/src/main/java/org/apache/fory/context/MetaStringReader.java
index 23b8d444e..7b492ddcb 100644
--- a/java/fory-core/src/main/java/org/apache/fory/context/MetaStringReader.java
+++ b/java/fory-core/src/main/java/org/apache/fory/context/MetaStringReader.java
@@ -23,11 +23,12 @@ import java.util.Arrays;
import org.apache.fory.annotation.Internal;
import org.apache.fory.collection.LongLongByteMap;
import org.apache.fory.collection.LongMap;
+import org.apache.fory.exception.ForyException;
import org.apache.fory.memory.LittleEndian;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.meta.EncodedMetaString;
+import org.apache.fory.meta.MetaString;
import org.apache.fory.resolver.SharedRegistry;
-import org.apache.fory.util.MurmurHash3;
/**
* Read-side state for meta-string references.
@@ -40,6 +41,9 @@ public final class MetaStringReader {
private static final int INITIAL_CAPACITY = 2;
private static final float LOAD_FACTOR = 0.5f;
private static final int SMALL_STRING_THRESHOLD = 16;
+ private static final int ENCODING_BITS = 4;
+ private static final int MAX_CACHED_READ_META_STRINGS = 8192;
+ private static final int MAX_CACHED_READ_META_STRING_LENGTH = 2048;
private final LongMap<EncodedMetaString> hash2MetaStringMap =
new LongMap<>(INITIAL_CAPACITY, LOAD_FACTOR);
@@ -47,7 +51,7 @@ public final class MetaStringReader {
new LongLongByteMap<>(INITIAL_CAPACITY, LOAD_FACTOR);
private final SharedRegistry sharedRegistry;
private EncodedMetaString[] dynamicReadStringIds = new EncodedMetaString[INITIAL_CAPACITY];
- private short dynamicReadStringId;
+ private int dynamicReadStringId;
/** Creates an empty reader state for one deserialization stream. */
public MetaStringReader(SharedRegistry sharedRegistry) {
@@ -68,7 +72,7 @@ public final class MetaStringReader {
updateDynamicString(encodedMetaString);
return encodedMetaString;
}
- return dynamicReadStringIds[len - 1];
+ return readDynamicString(len);
}
/**
@@ -88,7 +92,7 @@ public final class MetaStringReader {
updateDynamicString(encodedMetaString);
return encodedMetaString;
}
- return dynamicReadStringIds[len - 1];
+ return readDynamicString(len);
}
/** Reads a meta string from the current buffer, including any dynamic-id indirection. */
@@ -103,7 +107,7 @@ public final class MetaStringReader {
updateDynamicString(encodedMetaString);
return encodedMetaString;
}
- return dynamicReadStringIds[len - 1];
+ return readDynamicString(len);
}
/**
@@ -121,31 +125,75 @@ public final class MetaStringReader {
updateDynamicString(encodedMetaString);
return encodedMetaString;
}
- return dynamicReadStringIds[len - 1];
+ return readDynamicString(len);
}
private EncodedMetaString readBigMetaString(
MemoryBuffer buffer, EncodedMetaString cache, int len) {
long hashCode = buffer.readInt64();
- if (cache.hash == hashCode) {
- buffer.increaseReaderIndex(len);
+ if (cache.hash == hashCode && cache.bytes.length == len) {
+ // Big meta-string hashes are the wire identity on this cache hit. The body hash is computed
+ // and checked before a new entry is published; later hits intentionally skip the body.
+ buffer.checkReadableBytes(len);
+ buffer._increaseReaderIndexUnsafe(len);
return cache;
}
return readBigMetaString(buffer, len, hashCode);
}
private EncodedMetaString readBigMetaString(MemoryBuffer buffer, int len, long hashCode) {
+ buffer.checkReadableBytes(len);
EncodedMetaString encodedMetaString = hash2MetaStringMap.get(hashCode);
- if (encodedMetaString == null) {
- encodedMetaString =
- sharedRegistry.getOrCreateEncodedMetaString(buffer.readBytes(len), hashCode);
- hash2MetaStringMap.put(hashCode, encodedMetaString);
+ if (encodedMetaString != null && encodedMetaString.bytes.length == len) {
+ // Preserve the header-keyed fast path: entries reach this map only after their bytes matched
+ // the wire hash, so repeat hits advance over the redundant body without rehashing.
+ buffer._increaseReaderIndexUnsafe(len);
return encodedMetaString;
}
- buffer.increaseReaderIndex(len);
+ byte[] bytes = readAndValidateBigMetaString(buffer, len, hashCode);
+ EncodedMetaString canonicalMetaString =
+ sharedRegistry.getOrCreateEncodedMetaString(bytes, hashCode);
+ if (encodedMetaString == null
+ && len <= MAX_CACHED_READ_META_STRING_LENGTH
+ && hash2MetaStringMap.size < MAX_CACHED_READ_META_STRINGS) {
+ hash2MetaStringMap.put(hashCode, canonicalMetaString);
+ }
+ return canonicalMetaString;
+ }
+
+ private byte[] readAndValidateBigMetaString(MemoryBuffer buffer, int len, long hashCode) {
+ byte[] bytes = buffer.readBytes(len);
+ MetaString.Encoding encoding = MetaString.Encoding.fromInt((int) (hashCode & 0xff));
+ long canonicalHash = EncodedMetaString.computeHash(bytes, encoding);
+ if (canonicalHash != hashCode) {
+ throw new ForyException("Malformed meta string hash");
+ }
+ return bytes;
+ }
+
+ private boolean shouldCacheSmallMetaString() {
+ return longLongMetaStringMap.size < MAX_CACHED_READ_META_STRINGS;
+ }
+
+ private EncodedMetaString cacheSmallMetaString(
+ long v1, long v2, byte key, EncodedMetaString encodedMetaString) {
+ if (shouldCacheSmallMetaString()) {
+ longLongMetaStringMap.put(v1, v2, key, encodedMetaString);
+ }
return encodedMetaString;
}
+ private EncodedMetaString createSmallMetaString(
+ int len, MetaString.Encoding encoding, byte key, long v1, long v2) {
+ byte[] data = new byte[16];
+ LittleEndian.putInt64(data, 0, v1);
+ LittleEndian.putInt64(data, 8, v2);
+ byte[] bytes = Arrays.copyOf(data, len);
+ long hashCode = EncodedMetaString.computeHash(bytes, encoding);
+ return cacheSmallMetaString(
+ v1, v2, key, sharedRegistry.getOrCreateEncodedMetaString(bytes, hashCode));
+ }
+
private EncodedMetaString readSmallMetaString(MemoryBuffer buffer, int len) {
if (len == 0) {
return EncodedMetaString.EMPTY;
@@ -159,9 +207,11 @@ public final class MetaStringReader {
v1 = buffer.readInt64();
v2 = buffer.readBytesAsInt64(len - 8);
}
- EncodedMetaString encodedMetaString = longLongMetaStringMap.get(v1, v2, encoding);
+ int encodingValue = encoding & 0xff;
+ byte key = smallMetaStringKey(len, encodingValue);
+ EncodedMetaString encodedMetaString = longLongMetaStringMap.get(v1, v2, key);
if (encodedMetaString == null) {
- return createSmallMetaString(len, encoding, v1, v2);
+ return createSmallMetaString(len, MetaString.Encoding.fromInt(encodingValue), key, v1, v2);
}
return encodedMetaString;
}
@@ -180,39 +230,45 @@ public final class MetaStringReader {
v1 = buffer.readInt64();
v2 = buffer.readBytesAsInt64(len - 8);
}
- if (cache.first8Bytes == v1 && cache.second8Bytes == v2) {
+ int encodingValue = encoding & 0xff;
+ if (cache.bytes.length == len
+ && cache.encodingValue == encodingValue
+ && cache.first8Bytes == v1
+ && cache.second8Bytes == v2) {
return cache;
}
- EncodedMetaString encodedMetaString = longLongMetaStringMap.get(v1, v2, encoding);
+ byte key = smallMetaStringKey(len, encodingValue);
+ EncodedMetaString encodedMetaString = longLongMetaStringMap.get(v1, v2, key);
if (encodedMetaString == null) {
- return createSmallMetaString(len, encoding, v1, v2);
+ return createSmallMetaString(len, MetaString.Encoding.fromInt(encodingValue), key, v1, v2);
}
return encodedMetaString;
}
- private EncodedMetaString createSmallMetaString(int len, byte encoding, long v1, long v2) {
- byte[] data = new byte[16];
- LittleEndian.putInt64(data, 0, v1);
- LittleEndian.putInt64(data, 8, v2);
- long hashCode = MurmurHash3.murmurhash3_x64_128(data, 0, len, 47)[0];
- hashCode = Math.abs(hashCode);
- hashCode = (hashCode & 0xffffffffffffff00L) | encoding;
- EncodedMetaString encodedMetaString =
- sharedRegistry.getOrCreateEncodedMetaString(Arrays.copyOf(data, len), hashCode);
- longLongMetaStringMap.put(v1, v2, encoding, encodedMetaString);
- return encodedMetaString;
+ private static byte smallMetaStringKey(int len, int encodingValue) {
+ return (byte) (((len - 1) << ENCODING_BITS) | encodingValue);
+ }
+
+ private EncodedMetaString readDynamicString(int dynamicId) {
+ if (dynamicId <= 0 || dynamicId > dynamicReadStringId) {
+ throw new ForyException("Invalid meta string reference id " + dynamicId);
+ }
+ return dynamicReadStringIds[dynamicId - 1];
}
private void updateDynamicString(EncodedMetaString encodedMetaString) {
- short currentDynamicReadId = dynamicReadStringId++;
+ int currentDynamicReadId = dynamicReadStringId++;
EncodedMetaString[] readStringIds = dynamicReadStringIds;
if (readStringIds.length <= currentDynamicReadId) {
+ if (currentDynamicReadId >= MAX_CACHED_READ_META_STRINGS) {
+ throw new ForyException("Too many meta string references in payload");
+ }
readStringIds = dynamicReadStringIds = growRead(readStringIds, currentDynamicReadId);
}
readStringIds[currentDynamicReadId] = encodedMetaString;
}
- private EncodedMetaString[] growRead(EncodedMetaString[] current, int id) {
+ private static EncodedMetaString[] growRead(EncodedMetaString[] current, int id) {
int newLength = current.length;
while (newLength <= id) {
newLength <<= 1;
diff --git a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java
index 5def56c66..0b03800dd 100644
--- a/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java
+++ b/java/fory-core/src/main/java/org/apache/fory/context/ReadContext.java
@@ -467,11 +467,13 @@ public final class ReadContext {
} else {
size = buffer.readVarUInt32();
}
- if (buffer.readerIndex() + size > buffer.size() && buffer.getStreamReader() != null) {
- buffer.getStreamReader().fillBuffer(buffer.readerIndex() + size - buffer.size());
+ if (size < 0) {
+ throw new IllegalArgumentException("Buffer object size must be non-negative: " + size);
}
- MemoryBuffer slice = buffer.slice(buffer.readerIndex(), size);
- buffer.readerIndex(buffer.readerIndex() + size);
+ buffer.checkReadableBytes(size);
+ int readerIndex = buffer.readerIndex();
+ MemoryBuffer slice = buffer.slice(readerIndex, size);
+ buffer.readerIndex(readerIndex + size);
return slice;
}
Preconditions.checkArgument(outOfBandBuffers.hasNext());
diff --git a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java
index 1afbcbf7b..ff8d9f85a 100644
--- a/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java
+++ b/java/fory-core/src/main/java/org/apache/fory/io/ForyReadableChannel.java
@@ -66,9 +66,9 @@ public class ForyReadableChannel implements ForyStreamReader, ReadableByteChanne
memoryBuf.initDirectBuffer(ByteBufferUtil.getAddress(byteBuf), position, byteBuf);
}
byteBuf.limit(newLimit);
- int readCount = channel.read(byteBuf);
- memoryBuf.increaseSize(readCount);
- return readCount;
+ readFully(byteBuf, minFillSize);
+ memoryBuf.increaseSize(minFillSize);
+ return minFillSize;
} catch (IOException e) {
throw new DeserializationException("Failed to read the provided byte channel", e);
}
@@ -98,7 +98,7 @@ public class ForyReadableChannel implements ForyStreamReader, ReadableByteChanne
buf.readBytes(dst, dstIndex, remaining);
try {
ByteBuffer buffer = ByteBuffer.wrap(dst, dstIndex + remaining, length - remaining);
- channel.read(buffer);
+ readFully(buffer, length - remaining);
} catch (IOException e) {
throw new DeserializationException("Failed to read the provided byte channel", e);
}
@@ -130,10 +130,13 @@ public class ForyReadableChannel implements ForyStreamReader, ReadableByteChanne
int newLimit = dst.position() + length - remaining;
if (dstLimit > newLimit) {
dst.limit(newLimit);
- channel.read(dst);
- dst.limit(dstLimit);
+ try {
+ readFully(dst, length - remaining);
+ } finally {
+ dst.limit(dstLimit);
+ }
} else {
- channel.read(dst);
+ readFully(dst, length - remaining);
}
} catch (IOException e) {
throw new DeserializationException("Failed to read the provided byte channel", e);
@@ -169,4 +172,15 @@ public class ForyReadableChannel implements ForyStreamReader, ReadableByteChanne
public MemoryBuffer getBuffer() {
return memoryBuffer;
}
+
+ private void readFully(ByteBuffer dst, int length) throws IOException {
+ int remaining = length;
+ while (remaining > 0) {
+ int read = channel.read(dst);
+ if (read <= 0) {
+ throw new DeserializationException("Unexpected end of byte channel");
+ }
+ remaining -= read;
+ }
+ }
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
index 48d37e97d..89a0c96f8 100644
--- a/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/memory/MemoryBuffer.java
@@ -827,15 +827,15 @@ public final class MemoryBuffer {
}
private int continueWriteVarUInt32Small7(int value) {
- long encoded = (value & 0x7F);
+ int encoded = (value & 0x7F);
encoded |= (((value & 0x3f80) << 1) | 0x80);
int writerIdx = writerIndex;
if (value >>> 14 == 0) {
- _unsafePutInt32(writerIdx, (int) encoded);
+ _unsafePutInt32(writerIdx, encoded);
writerIndex += 2;
return 2;
}
- int diff = continuePutVarInt36(writerIdx, encoded, value);
+ int diff = continuePutVarUInt32(writerIdx, encoded, value);
writerIndex += diff;
return diff;
}
@@ -1811,7 +1811,7 @@ public final class MemoryBuffer {
int readIdx = readerIndex;
int result;
if (size - readIdx < 5) {
- result = (int) readVarUint36Slow();
+ result = readVarUInt32Slow();
} else {
long address = this.address;
// | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits |
@@ -1835,7 +1835,11 @@ public final class MemoryBuffer {
// 0xfe00000: 0b1111111 << 21
result |= (fourByteValue >>> 3) & 0xfe00000;
if ((fourByteValue & 0x80000000) != 0) {
- result |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F) << 28;
+ int fifthByte = UNSAFE.getByte(heapMemory, address + readIdx++) & 0xFF;
+ if ((fifthByte & 0xF0) != 0) {
+ throwMalformedVarUInt32(fifthByte);
+ }
+ result |= fifthByte << 28;
}
}
}
@@ -1854,7 +1858,7 @@ public final class MemoryBuffer {
int readIdx = readerIndex;
int result;
if (size - readIdx < 5) {
- result = (int) readVarUint36Slow();
+ result = readVarUInt32Slow();
} else {
long address = this.address;
int fourByteValue = Integer.reverseBytes(UNSAFE.getInt(heapMemory, address + readIdx));
@@ -1877,7 +1881,11 @@ public final class MemoryBuffer {
// 0xfe00000: 0b1111111 << 21
result |= (fourByteValue >>> 3) & 0xfe00000;
if ((fourByteValue & 0x80000000) != 0) {
- result |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F) << 28;
+ int fifthByte = UNSAFE.getByte(heapMemory, address + readIdx++) & 0xFF;
+ if ((fifthByte & 0xF0) != 0) {
+ throwMalformedVarUInt32(fifthByte);
+ }
+ result |= fifthByte << 28;
}
}
}
@@ -1956,11 +1964,45 @@ public final class MemoryBuffer {
return result;
}
+ private int readVarUInt32Slow() {
+ int b = readByte() & 0xFF;
+ int result = b & 0x7F;
+ // Note:
+ // Loop are not used here to improve performance.
+ // We manually unroll the loop for better performance.
+ // noinspection Duplicates
+ if ((b & 0x80) != 0) {
+ b = readByte() & 0xFF;
+ result |= (b & 0x7F) << 7;
+ if ((b & 0x80) != 0) {
+ b = readByte() & 0xFF;
+ result |= (b & 0x7F) << 14;
+ if ((b & 0x80) != 0) {
+ b = readByte() & 0xFF;
+ result |= (b & 0x7F) << 21;
+ if ((b & 0x80) != 0) {
+ b = readByte() & 0xFF;
+ if ((b & 0xF0) != 0) {
+ throwMalformedVarUInt32(b);
+ }
+ result |= b << 28;
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ private static void throwMalformedVarUInt32(int fifthByte) {
+ throw new IllegalArgumentException(
+ "Malformed varuint32 fifth byte " + fifthByte + " exceeds 32 bits");
+ }
+
/** Reads the 1-5 byte int part of a non-negative varint. */
public int readVarUInt32() {
int readIdx = readerIndex;
if (size - readIdx < 5) {
- return (int) readVarUint36Slow();
+ return readVarUInt32Slow();
}
// | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits | 1bit + 7bits |
int fourByteValue = _unsafeGetInt32(readIdx);
@@ -1983,7 +2025,11 @@ public final class MemoryBuffer {
// 0xfe00000: 0b1111111 << 21
result |= (fourByteValue >>> 3) & 0xfe00000;
if ((fourByteValue & 0x80000000) != 0) {
- result |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F) << 28;
+ int fifthByte = UNSAFE.getByte(heapMemory, address + readIdx++) & 0xFF;
+ if ((fifthByte & 0xF0) != 0) {
+ throwMalformedVarUInt32(fifthByte);
+ }
+ result |= fifthByte << 28;
}
}
}
@@ -2031,7 +2077,7 @@ public final class MemoryBuffer {
readerIndex = readIdx;
return value;
} else {
- return (int) readVarUint36Slow();
+ return readVarUInt32Slow();
}
}
@@ -2044,7 +2090,11 @@ public final class MemoryBuffer {
readIdx++;
value |= (bulkRead >>> 3) & 0xfe00000;
if ((bulkRead & 0x80000000) != 0) {
- value |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F) << 28;
+ int fifthByte = UNSAFE.getByte(heapMemory, address + readIdx++) & 0xFF;
+ if ((fifthByte & 0xF0) != 0) {
+ throwMalformedVarUInt32(fifthByte);
+ }
+ value |= fifthByte << 28;
}
}
readerIndex = readIdx;
@@ -2440,7 +2490,7 @@ public final class MemoryBuffer {
}
readerIndex = readIdx;
} else {
- binarySize = (int) readVarUint36Slow();
+ binarySize = readVarUInt32Slow();
readIdx = readerIndex;
}
int diff = size - readIdx;
@@ -2459,7 +2509,11 @@ public final class MemoryBuffer {
readIdx++;
binarySize |= (bulkRead >>> 3) & 0xfe00000;
if ((bulkRead & 0x80000000) != 0) {
- binarySize |= (UNSAFE.getByte(heapMemory, address + readIdx++) & 0x7F) << 28;
+ int fifthByte = UNSAFE.getByte(heapMemory, address + readIdx++) & 0xFF;
+ if ((fifthByte & 0xF0) != 0) {
+ throwMalformedVarUInt32(fifthByte);
+ }
+ binarySize |= fifthByte << 28;
}
}
int diff = size - readIdx;
@@ -2716,6 +2770,28 @@ public final class MemoryBuffer {
return Platform.arrayEquals(heapMemory, pos1, buf2.heapMemory, pos2, len);
}
+ /**
+ * Equals a memory buffer region with a byte array region.
+ *
+ * @param bytes Array to compare with
+ * @param bytesOffset Offset of bytes to start comparing
+ * @param offset Offset of this buffer to start comparing
+ * @param len Length of the compared memory region
+ * @return true if regions are equal or len zero, false otherwise
+ */
+ public boolean equalTo(byte[] bytes, int bytesOffset, int offset, int len) {
+ checkArgument(bytes != null);
+ checkArgument(len >= 0);
+ checkArgument(bytesOffset >= 0 && bytesOffset <= bytes.length - len);
+ checkArgument(offset >= 0 && offset <= size - len);
+ if (len == 0) {
+ return true;
+ }
+ final long pos = address + offset;
+ return Platform.arrayEquals(
+ heapMemory, pos, bytes, Platform.BYTE_ARRAY_OFFSET + bytesOffset, len);
+ }
+
@Override
public String toString() {
return "MemoryBuffer{"
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/DeflaterMetaCompressor.java b/java/fory-core/src/main/java/org/apache/fory/meta/DeflaterMetaCompressor.java
index 761174112..cf8cee268 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/DeflaterMetaCompressor.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/DeflaterMetaCompressor.java
@@ -43,6 +43,11 @@ public class DeflaterMetaCompressor implements MetaCompressor {
@Override
public byte[] decompress(byte[] input, int offset, int size) {
+ return decompress(input, offset, size, Integer.MAX_VALUE);
+ }
+
+ @Override
+ public byte[] decompress(byte[] input, int offset, int size, int maxOutputSize) {
Inflater inflater = new Inflater();
inflater.setInput(input, offset, size);
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@@ -51,6 +56,10 @@ public class DeflaterMetaCompressor implements MetaCompressor {
while (!inflater.finished()) {
int decompressedSize = inflater.inflate(buffer);
if (decompressedSize > 0) {
+ if (outputStream.size() > maxOutputSize - decompressedSize) {
+ throw new InvalidDataException(
+ "Decompressed TypeDef metadata exceeds the maximum size.");
+ }
outputStream.write(buffer, 0, decompressedSize);
continue;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/EncodedMetaString.java b/java/fory-core/src/main/java/org/apache/fory/meta/EncodedMetaString.java
index b13c1630b..1692aa63d 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/EncodedMetaString.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/EncodedMetaString.java
@@ -33,6 +33,7 @@ public final class EncodedMetaString {
public final byte[] bytes;
public final long hash;
+ public final int encodingValue;
public final MetaString.Encoding encoding;
public final long first8Bytes;
public final long second8Bytes;
@@ -45,7 +46,8 @@ public final class EncodedMetaString {
assert hash != 0;
this.bytes = bytes;
this.hash = hash;
- this.encoding = MetaString.Encoding.fromInt((int) (hash & HEADER_MASK));
+ this.encodingValue = (int) (hash & HEADER_MASK);
+ this.encoding = MetaString.Encoding.fromInt(encodingValue);
byte[] data = bytes;
if (bytes.length < 16) {
data = new byte[16];
@@ -55,7 +57,7 @@ public final class EncodedMetaString {
second8Bytes = LittleEndian.getInt64(data, Platform.BYTE_ARRAY_OFFSET + 8);
}
- private static long computeHash(byte[] bytes, MetaString.Encoding encoding) {
+ public static long computeHash(byte[] bytes, MetaString.Encoding encoding) {
long hash = MurmurHash3.murmurhash3_x64_128(bytes, 0, bytes.length, 47)[0];
hash = Math.abs(hash);
if (hash == 0) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/MetaCompressor.java b/java/fory-core/src/main/java/org/apache/fory/meta/MetaCompressor.java
index 0ed3974b1..cf74e20c2 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/MetaCompressor.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/MetaCompressor.java
@@ -19,6 +19,8 @@
package org.apache.fory.meta;
+import org.apache.fory.exception.InvalidDataException;
+
/**
* An interface used to compress class metadata such as field names and types. The implementation of
* this interface should be thread safe.
@@ -28,6 +30,14 @@ public interface MetaCompressor {
byte[] decompress(byte[] data, int offset, int size);
+ default byte[] decompress(byte[] data, int offset, int size, int maxOutputSize) {
+ byte[] decompressed = decompress(data, offset, size);
+ if (decompressed.length > maxOutputSize) {
+ throw new InvalidDataException("Decompressed TypeDef metadata exceeds the maximum size.");
+ }
+ return decompressed;
+ }
+
/**
* Check whether {@link MetaCompressor} implements `equals/hashCode` method. If not implemented,
* return {@link TypeEqualMetaCompressor} instead which compare equality by the compressor type
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java
index f3c02af17..b4f50c2c0 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefDecoder.java
@@ -25,12 +25,12 @@ import static org.apache.fory.meta.Encoders.typeNameEncodings;
import static org.apache.fory.meta.NativeTypeDefEncoder.BIG_NAME_THRESHOLD;
import static org.apache.fory.meta.NativeTypeDefEncoder.NUM_CLASS_THRESHOLD;
import static org.apache.fory.meta.TypeDef.COMPRESS_META_FLAG;
-import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG;
import static org.apache.fory.meta.TypeDef.META_SIZE_MASKS;
import java.util.ArrayList;
import java.util.List;
import org.apache.fory.collection.Tuple2;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.meta.FieldTypes.FieldType;
import org.apache.fory.meta.MetaString.Encoding;
@@ -38,6 +38,7 @@ import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.resolver.TypeResolver;
import org.apache.fory.serializer.UnknownClass;
import org.apache.fory.type.Types;
+import org.apache.fory.util.MurmurHash3;
import org.apache.fory.util.Preconditions;
/**
@@ -46,8 +47,13 @@ import org.apache.fory.util.Preconditions;
* href="https://fory.apache.org/docs/specification/fory_java_serialization_spec">...</a>
*/
class NativeTypeDefDecoder {
+ private static final int MAX_TYPE_DEF_SIZE_BYTES = 16 * 1024 * 1024;
+
static Tuple2<byte[], byte[]> decodeTypeDefBuf(
MemoryBuffer inputBuffer, TypeResolver resolver, long id) {
+ if ((id & TypeDef.RESERVED_META_FLAGS) != 0) {
+ throw new DeserializationException("Invalid TypeDef global header");
+ }
MemoryBuffer encoded = MemoryBuffer.newHeapBuffer(64);
encoded.writeInt64(id);
int size = (int) (id & META_SIZE_MASKS);
@@ -56,10 +62,17 @@ class NativeTypeDefDecoder {
encoded.writeVarUInt32(moreSize);
size += moreSize;
}
+ if (size > MAX_TYPE_DEF_SIZE_BYTES) {
+ throw new DeserializationException("TypeDef metadata size exceeds the maximum size");
+ }
byte[] encodedTypeDef = inputBuffer.readBytes(size);
encoded.writeBytes(encodedTypeDef);
if ((id & COMPRESS_META_FLAG) != 0) {
- encodedTypeDef = resolver.getConfig().getMetaCompressor().decompress(encodedTypeDef, 0, size);
+ encodedTypeDef =
+ resolver
+ .getConfig()
+ .getMetaCompressor()
+ .decompress(encodedTypeDef, 0, size, MAX_TYPE_DEF_SIZE_BYTES);
}
return Tuple2.of(encodedTypeDef, encoded.getBytes(0, encoded.writerIndex()));
}
@@ -67,7 +80,9 @@ class NativeTypeDefDecoder {
public static TypeDef decodeTypeDef(ClassResolver resolver, MemoryBuffer buffer, long id) {
Tuple2<byte[], byte[]> decoded = decodeTypeDefBuf(buffer, resolver, id);
MemoryBuffer typeDefBuf = MemoryBuffer.fromByteArray(decoded.f0);
- int numClasses = typeDefBuf.readByte();
+ int bodyHeader = typeDefBuf.readByte() & 0xff;
+ int rootTypeId = nativeTypeId(bodyHeader >>> 4);
+ int numClasses = bodyHeader & NUM_CLASS_THRESHOLD;
if (numClasses == NUM_CLASS_THRESHOLD) {
numClasses += typeDefBuf.readVarUInt32Small7();
}
@@ -75,12 +90,15 @@ class NativeTypeDefDecoder {
String className;
List<FieldInfo> classFields = new ArrayList<>();
ClassSpec classSpec = null;
+ Class<?> rootClass = null;
+ boolean rootClassLayerRegistered = false;
for (int i = 0; i < numClasses; i++) {
// | num fields + register flag | header + package name | header + class name
// | header + type id + field name | next field info | ... |
int currentClassHeader = typeDefBuf.readVarUInt32Small7();
boolean isRegistered = (currentClassHeader & 0b1) != 0;
int numFields = currentClassHeader >>> 1;
+ Class<?> currentClass = null;
if (isRegistered) {
int typeId = typeDefBuf.readUInt8();
int userTypeId = -1;
@@ -89,11 +107,16 @@ class NativeTypeDefDecoder {
}
Class<?> cls = resolver.getRegisteredClassByTypeId(typeId, userTypeId);
if (cls == null) {
- classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, userTypeId);
+ classSpec =
+ new ClassSpec(
+ UnknownClass.UnknownStruct.class,
+ i == numClasses - 1 ? rootTypeId : typeId,
+ userTypeId);
className = classSpec.entireClassName;
} else {
className = cls.getName();
- classSpec = new ClassSpec(cls, typeId, userTypeId);
+ classSpec = new ClassSpec(cls, i == numClasses - 1 ? rootTypeId : typeId, userTypeId);
+ currentClass = cls;
}
} else {
String pkg = readPkgName(typeDefBuf);
@@ -103,20 +126,22 @@ class NativeTypeDefDecoder {
if (resolver.isRegisteredByName(className)) {
Class<?> cls = resolver.getRegisteredClass(className);
className = cls.getName();
- classSpec =
- new ClassSpec(
- cls, resolver.getTypeIdForTypeDef(cls), resolver.getUserTypeIdForTypeDef(cls));
+ int typeId = i == numClasses - 1 ? rootTypeId : resolver.getTypeIdForTypeDef(cls);
+ classSpec = new ClassSpec(cls, typeId, resolver.getUserTypeIdForTypeDef(cls));
+ currentClass = cls;
} else {
Class<?> cls =
resolver.loadClassForMeta(
decodedSpec.entireClassName, decodedSpec.isEnum, decodedSpec.dimension);
if (UnknownClass.isUnknowClass(cls)) {
- int typeId;
+ int decodedTypeId;
if (decodedSpec.isEnum) {
- typeId = Types.NAMED_ENUM;
+ decodedTypeId = Types.NAMED_ENUM;
} else {
- typeId = resolver.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT;
+ decodedTypeId =
+ resolver.isCompatible() ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT;
}
+ int typeId = i == numClasses - 1 ? rootTypeId : decodedTypeId;
classSpec =
new ClassSpec(
decodedSpec.entireClassName,
@@ -128,18 +153,120 @@ class NativeTypeDefDecoder {
classSpec.type = cls;
className = classSpec.entireClassName;
} else {
- int typeId = resolver.getTypeIdForTypeDef(cls);
+ int typeId = i == numClasses - 1 ? rootTypeId : resolver.getTypeIdForTypeDef(cls);
classSpec = new ClassSpec(cls, typeId, resolver.getUserTypeIdForTypeDef(cls));
className = classSpec.entireClassName;
+ currentClass = cls;
}
}
}
+ if (i == numClasses - 1) {
+ rootClass = currentClass;
+ rootClassLayerRegistered = isRegistered;
+ }
List<FieldInfo> fieldInfos = readFieldsInfo(typeDefBuf, resolver, className, numFields);
classFields.addAll(fieldInfos);
}
Preconditions.checkNotNull(classSpec);
- boolean hasFieldsMeta = (id & HAS_FIELDS_META_FLAG) != 0;
- return new TypeDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1);
+ boolean hasFieldMetadata = !classFields.isEmpty();
+ if (!Types.isStructType(rootTypeId) && hasFieldMetadata) {
+ throw new DeserializationException("Non-struct TypeDef cannot carry field metadata");
+ }
+ if (rootClass != null) {
+ int expectedRootTypeId = resolver.getTypeDefRootTypeId(rootClass, hasFieldMetadata);
+ if (!isCompatibleRootKind(expectedRootTypeId, rootTypeId, !rootClassLayerRegistered)) {
+ throw new DeserializationException(
+ "TypeDef root kind does not match the decoded class: class="
+ + rootClass.getName()
+ + ", expected="
+ + expectedRootTypeId
+ + ", actual="
+ + rootTypeId
+ + ", registeredClassLayer="
+ + rootClassLayerRegistered);
+ }
+ }
+ if (typeDefBuf.remaining() != 0) {
+ throw new DeserializationException("Invalid TypeDef metadata size");
+ }
+ validateParsedTypeDefHash(id, decoded.f1);
+ return new TypeDef(classSpec, classFields, id, decoded.f1);
+ }
+
+ private static boolean isCompatibleRootKind(
+ int expectedTypeId, int actualTypeId, boolean allowNamednessDifference) {
+ if (expectedTypeId == actualTypeId) {
+ return true;
+ }
+ if (allowNamednessDifference) {
+ return Types.isStructType(expectedTypeId) && Types.isStructType(actualTypeId);
+ }
+ return isStructCompatibilityVariant(expectedTypeId, actualTypeId);
+ }
+
+ private static boolean isStructCompatibilityVariant(int expectedTypeId, int actualTypeId) {
+ boolean expectedIdStruct =
+ expectedTypeId == Types.STRUCT || expectedTypeId == Types.COMPATIBLE_STRUCT;
+ boolean actualIdStruct =
+ actualTypeId == Types.STRUCT || actualTypeId == Types.COMPATIBLE_STRUCT;
+ if (expectedIdStruct || actualIdStruct) {
+ return expectedIdStruct && actualIdStruct;
+ }
+ boolean expectedNamedStruct =
+ expectedTypeId == Types.NAMED_STRUCT || expectedTypeId == Types.NAMED_COMPATIBLE_STRUCT;
+ boolean actualNamedStruct =
+ actualTypeId == Types.NAMED_STRUCT || actualTypeId == Types.NAMED_COMPATIBLE_STRUCT;
+ return expectedNamedStruct && actualNamedStruct;
+ }
+
+ static int nativeTypeId(int kindCode) {
+ switch (kindCode) {
+ case 0:
+ return Types.STRUCT;
+ case 1:
+ return Types.COMPATIBLE_STRUCT;
+ case 2:
+ return Types.NAMED_STRUCT;
+ case 3:
+ return Types.NAMED_COMPATIBLE_STRUCT;
+ case 4:
+ return Types.ENUM;
+ case 5:
+ return Types.NAMED_ENUM;
+ case 6:
+ return Types.EXT;
+ case 7:
+ return Types.NAMED_EXT;
+ case 8:
+ return Types.TYPED_UNION;
+ case 9:
+ return Types.NAMED_UNION;
+ default:
+ throw new DeserializationException("Unsupported TypeDef kind code " + kindCode);
+ }
+ }
+
+ static void validateParsedTypeDefHash(long id, byte[] encoded) {
+ int size = (int) (id & META_SIZE_MASKS);
+ int bodyOffset = Long.BYTES;
+ if (size == META_SIZE_MASKS) {
+ MemoryBuffer encodedBuffer = MemoryBuffer.fromByteArray(encoded);
+ encodedBuffer.readerIndex(Long.BYTES);
+ int moreSize = encodedBuffer.readVarUInt32Small14();
+ size += moreSize;
+ bodyOffset = encodedBuffer.readerIndex();
+ }
+ if (encoded.length - bodyOffset != size) {
+ throw new DeserializationException("Invalid TypeDef encoded size");
+ }
+ long hash = MurmurHash3.murmurhash3_x64_128(encoded, bodyOffset, size, 47)[0];
+ hash <<= (Long.SIZE - TypeDef.NUM_HASH_BITS);
+ long hashMask = -1L << (Long.SIZE - TypeDef.NUM_HASH_BITS);
+ long expectedHeaderHash = Math.abs(hash) & hashMask;
+ long actualHeaderHash = id & hashMask;
+ if (expectedHeaderHash != actualHeaderHash) {
+ throw new DeserializationException("Invalid TypeDef metadata hash");
+ }
}
private static List<FieldInfo> readFieldsInfo(
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java
index 3f7998184..6173a9e88 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/NativeTypeDefEncoder.java
@@ -23,7 +23,6 @@ import static org.apache.fory.meta.Encoders.fieldNameEncodingsList;
import static org.apache.fory.meta.Encoders.pkgEncodingsList;
import static org.apache.fory.meta.Encoders.typeNameEncodingsList;
import static org.apache.fory.meta.TypeDef.COMPRESS_META_FLAG;
-import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG;
import static org.apache.fory.meta.TypeDef.META_SIZE_MASKS;
import static org.apache.fory.meta.TypeDef.NUM_HASH_BITS;
@@ -56,7 +55,6 @@ import org.apache.fory.util.MurmurHash3;
*/
@Internal
public class NativeTypeDefEncoder {
- // a flag to mark a type is not struct.
static final int NUM_CLASS_THRESHOLD = 0b1111;
private static final java.util.function.Function<Descriptor, Descriptor> IDENTITY_DESCRIPTOR =
descriptor -> descriptor;
@@ -142,43 +140,45 @@ public class NativeTypeDefEncoder {
}
/** Build class definition from fields of class. */
- static TypeDef buildTypeDef(
- ClassResolver classResolver, Class<?> type, List<Field> fields, boolean hasFieldsMeta) {
- return buildTypeDefWithFieldInfos(
- classResolver, type, buildFieldsInfo(classResolver, fields), hasFieldsMeta);
+ static TypeDef buildTypeDef(ClassResolver classResolver, Class<?> type, List<Field> fields) {
+ return buildTypeDefWithFieldInfos(classResolver, type, buildFieldsInfo(classResolver, fields));
}
public static TypeDef buildTypeDefWithFieldInfos(
- ClassResolver classResolver,
- Class<?> type,
- List<FieldInfo> fieldInfos,
- boolean hasFieldsMeta) {
+ ClassResolver classResolver, Class<?> type, List<FieldInfo> fieldInfos) {
+ boolean hasFieldMetadata = !fieldInfos.isEmpty();
Map<String, List<FieldInfo>> classLayers = getClassFields(type, fieldInfos);
fieldInfos = new ArrayList<>(fieldInfos.size());
classLayers.values().forEach(fieldInfos::addAll);
- MemoryBuffer encodeTypeDef = encodeTypeDef(classResolver, type, classLayers, hasFieldsMeta);
+ MemoryBuffer encodeTypeDef = encodeTypeDef(classResolver, type, classLayers, hasFieldMetadata);
byte[] typeDefBytes = encodeTypeDef.getBytes(0, encodeTypeDef.writerIndex());
- int typeId = classResolver.getTypeIdForTypeDef(type);
+ int typeId = classResolver.getTypeDefRootTypeId(type, hasFieldMetadata);
int userTypeId = classResolver.getUserTypeIdForTypeDef(type);
ClassSpec classSpec = new ClassSpec(type, typeId, userTypeId);
- return new TypeDef(
- classSpec, fieldInfos, hasFieldsMeta, encodeTypeDef.getInt64(0), typeDefBytes);
+ return new TypeDef(classSpec, fieldInfos, encodeTypeDef.getInt64(0), typeDefBytes);
}
// see spec documentation: docs/specification/java_serialization_spec.md
// https://fory.apache.org/docs/specification/fory_java_serialization_spec
public static MemoryBuffer encodeTypeDef(
+ ClassResolver classResolver, Class<?> type, Map<String, List<FieldInfo>> classLayers) {
+ return encodeTypeDef(classResolver, type, classLayers, hasFieldMetadata(classLayers));
+ }
+
+ private static MemoryBuffer encodeTypeDef(
ClassResolver classResolver,
Class<?> type,
Map<String, List<FieldInfo>> classLayers,
- boolean hasFieldsMeta) {
+ boolean hasFieldMetadata) {
MemoryBuffer typeDefBuf = MemoryBuffer.newHeapBuffer(128);
int numClasses = classLayers.size() - 1; // num class must be greater than 0
+ int rootTypeId = classResolver.getTypeDefRootTypeId(type, hasFieldMetadata);
+ int firstBodyByte = nativeKindCode(rootTypeId) << 4;
if (numClasses >= NUM_CLASS_THRESHOLD) {
- typeDefBuf.writeByte(NUM_CLASS_THRESHOLD);
+ typeDefBuf.writeByte(firstBodyByte | NUM_CLASS_THRESHOLD);
typeDefBuf.writeVarUInt32Small7(numClasses - NUM_CLASS_THRESHOLD);
} else {
- typeDefBuf.writeByte(numClasses);
+ typeDefBuf.writeByte(firstBodyByte | numClasses);
}
for (Map.Entry<String, List<FieldInfo>> entry : classLayers.entrySet()) {
String className = entry.getKey();
@@ -224,11 +224,19 @@ public class NativeTypeDefEncoder {
typeDefBuf = MemoryBuffer.fromByteArray(compressed);
typeDefBuf.writerIndex(compressed.length);
}
- return prependHeader(typeDefBuf, isCompressed, hasFieldsMeta);
+ return prependHeader(typeDefBuf, isCompressed);
+ }
+
+ private static boolean hasFieldMetadata(Map<String, List<FieldInfo>> classLayers) {
+ for (List<FieldInfo> fields : classLayers.values()) {
+ if (!fields.isEmpty()) {
+ return true;
+ }
+ }
+ return false;
}
- static MemoryBuffer prependHeader(
- MemoryBuffer buffer, boolean isCompressed, boolean hasFieldsMeta) {
+ static MemoryBuffer prependHeader(MemoryBuffer buffer, boolean isCompressed) {
int metaSize = buffer.writerIndex();
long hash = MurmurHash3.murmurhash3_x64_128(buffer.getHeapMemory(), 0, metaSize, 47)[0];
hash <<= (64 - NUM_HASH_BITS);
@@ -237,9 +245,6 @@ public class NativeTypeDefEncoder {
if (isCompressed) {
header |= COMPRESS_META_FLAG;
}
- if (hasFieldsMeta) {
- header |= HAS_FIELDS_META_FLAG;
- }
header |= Math.min(metaSize, META_SIZE_MASKS);
MemoryBuffer result = MemoryUtils.buffer(metaSize + 8);
result.writeInt64(header);
@@ -250,6 +255,33 @@ public class NativeTypeDefEncoder {
return result;
}
+ static int nativeKindCode(int typeId) {
+ switch (typeId) {
+ case Types.STRUCT:
+ return 0;
+ case Types.COMPATIBLE_STRUCT:
+ return 1;
+ case Types.NAMED_STRUCT:
+ return 2;
+ case Types.NAMED_COMPATIBLE_STRUCT:
+ return 3;
+ case Types.ENUM:
+ return 4;
+ case Types.NAMED_ENUM:
+ return 5;
+ case Types.EXT:
+ return 6;
+ case Types.NAMED_EXT:
+ return 7;
+ case Types.TYPED_UNION:
+ return 8;
+ case Types.NAMED_UNION:
+ return 9;
+ default:
+ throw new IllegalArgumentException("Unsupported TypeDef kind " + typeId);
+ }
+ }
+
private static Class<?> getType(Class<?> cls, String type) {
Class<?> c = cls;
while (cls != null) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java
index 4fd91857c..6069472e9 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDef.java
@@ -34,6 +34,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import org.apache.fory.builder.MetaSharedCodecBuilder;
import org.apache.fory.config.ForyBuilder;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.logging.Logger;
import org.apache.fory.logging.LoggerFactory;
import org.apache.fory.memory.MemoryBuffer;
@@ -69,11 +70,11 @@ import org.apache.fory.util.StringUtils;
public class TypeDef implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(TypeDef.class);
- static final int COMPRESS_META_FLAG = 0b1 << 9;
- static final int HAS_FIELDS_META_FLAG = 0b1 << 8;
+ static final int COMPRESS_META_FLAG = 0b1 << 8;
+ static final long RESERVED_META_FLAGS = 0b111L << 9;
// low 8 bits
static final int META_SIZE_MASKS = 0xff;
- static final int NUM_HASH_BITS = 50;
+ static final int NUM_HASH_BITS = 52;
// TODO use field offset to sort field, which will hit l1-cache more. Since
// `objectFieldOffset` is not part of jvm-specification, it may change between different jdk
@@ -101,30 +102,33 @@ public class TypeDef implements Serializable {
private final ClassSpec classSpec;
private final List<FieldInfo> fieldsInfo;
- private final boolean hasFieldsMeta;
// Unique id for class def. If class def are same between processes, then the id will
// be same too.
private final long id;
private final byte[] encoded;
- TypeDef(
- ClassSpec classSpec,
- List<FieldInfo> fieldsInfo,
- boolean hasFieldsMeta,
- long id,
- byte[] encoded) {
+ TypeDef(ClassSpec classSpec, List<FieldInfo> fieldsInfo, long id, byte[] encoded) {
this.classSpec = classSpec;
this.fieldsInfo = fieldsInfo;
- this.hasFieldsMeta = hasFieldsMeta;
this.id = id;
this.encoded = encoded;
}
public static void skipTypeDef(MemoryBuffer buffer, long id) {
+ // Header-cache hits intentionally treat the current body as opaque bytes and skip by the size
+ // in
+ // the current header. Parsed TypeDefs are published to the cache only after successful body
+ // parse
+ // and 52-bit body-hash validation; cache hits must not reparse or rehash that body.
int size = (int) (id & META_SIZE_MASKS);
if (size == META_SIZE_MASKS) {
- size += buffer.readVarUInt32Small14();
+ int extendedSize = buffer.readVarUInt32Small14();
+ if (extendedSize < 0 || extendedSize > Integer.MAX_VALUE - size) {
+ throw new DeserializationException("Invalid TypeDef metadata size " + extendedSize);
+ }
+ size += extendedSize;
}
+ buffer.checkReadableBytes(size);
buffer.increaseReaderIndex(size);
}
@@ -146,11 +150,6 @@ public class TypeDef implements Serializable {
return fieldsInfo;
}
- /** Returns ext meta for the class. */
- public boolean hasFieldsMeta() {
- return hasFieldsMeta;
- }
-
/**
* Returns an unique id for class def. If class def are same between processes, then the id will
* be same too.
@@ -179,6 +178,10 @@ public class TypeDef implements Serializable {
|| classSpec.typeId == Types.NAMED_COMPATIBLE_STRUCT;
}
+ public boolean isStructSchemaKind() {
+ return Types.isStructType(classSpec.typeId);
+ }
+
public int getUserTypeId() {
Preconditions.checkArgument(!isNamed(), "Named types don't have user type id");
return classSpec.userTypeId;
@@ -190,8 +193,7 @@ public class TypeDef implements Serializable {
return false;
}
TypeDef typeDef = (TypeDef) o;
- return hasFieldsMeta == typeDef.hasFieldsMeta
- && id == typeDef.id
+ return id == typeDef.id
&& Objects.equals(classSpec, typeDef.classSpec)
&& Objects.equals(fieldsInfo, typeDef.fieldsInfo);
}
@@ -209,8 +211,6 @@ public class TypeDef implements Serializable {
+ '\''
+ ", fieldsInfo="
+ fieldsInfo
- + ", hasFieldsMeta="
- + hasFieldsMeta
+ ", id="
+ id
+ '}';
@@ -429,17 +429,12 @@ public class TypeDef implements Serializable {
return TypeDefEncoder.buildTypeDef((XtypeResolver) resolver, cls);
}
return NativeTypeDefEncoder.buildTypeDef(
- (ClassResolver) resolver, cls, buildFields(resolver, cls, resolveParent), true);
+ (ClassResolver) resolver, cls, buildFields(resolver, cls, resolveParent));
}
/** Build class definition from fields of class. */
static TypeDef buildTypeDef(ClassResolver classResolver, Class<?> type, List<Field> fields) {
- return buildTypeDef(classResolver, type, fields, true);
- }
-
- public static TypeDef buildTypeDef(
- ClassResolver classResolver, Class<?> type, List<Field> fields, boolean hasFieldsMeta) {
- return NativeTypeDefEncoder.buildTypeDef(classResolver, type, fields, hasFieldsMeta);
+ return NativeTypeDefEncoder.buildTypeDef(classResolver, type, fields);
}
public TypeDef replaceRootClassTo(TypeResolver resolver, Class<?> targetCls) {
@@ -460,6 +455,6 @@ public class TypeDef implements Serializable {
(XtypeResolver) resolver, targetCls, fieldInfos);
}
return NativeTypeDefEncoder.buildTypeDefWithFieldInfos(
- (ClassResolver) resolver, targetCls, fieldInfos, hasFieldsMeta);
+ (ClassResolver) resolver, targetCls, fieldInfos);
}
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java
index 296316b8f..b184656c4 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefDecoder.java
@@ -23,14 +23,18 @@ import static org.apache.fory.meta.Encoders.fieldNameEncodings;
import static org.apache.fory.meta.NativeTypeDefDecoder.decodeTypeDefBuf;
import static org.apache.fory.meta.NativeTypeDefDecoder.readPkgName;
import static org.apache.fory.meta.NativeTypeDefDecoder.readTypeName;
-import static org.apache.fory.meta.TypeDef.HAS_FIELDS_META_FLAG;
+import static org.apache.fory.meta.NativeTypeDefDecoder.validateParsedTypeDefHash;
+import static org.apache.fory.meta.TypeDef.COMPRESS_META_FLAG;
+import static org.apache.fory.meta.TypeDefEncoder.COMPATIBLE_FLAG;
import static org.apache.fory.meta.TypeDefEncoder.FIELD_NAME_SIZE_THRESHOLD;
import static org.apache.fory.meta.TypeDefEncoder.REGISTER_BY_NAME_FLAG;
import static org.apache.fory.meta.TypeDefEncoder.SMALL_NUM_FIELDS_THRESHOLD;
+import static org.apache.fory.meta.TypeDefEncoder.STRUCT_FLAG;
import java.util.ArrayList;
import java.util.List;
import org.apache.fory.collection.Tuple2;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.logging.Logger;
import org.apache.fory.logging.LoggerFactory;
import org.apache.fory.memory.MemoryBuffer;
@@ -39,12 +43,12 @@ import org.apache.fory.meta.MetaString.Encoding;
import org.apache.fory.resolver.TypeInfo;
import org.apache.fory.resolver.XtypeResolver;
import org.apache.fory.serializer.UnknownClass;
+import org.apache.fory.type.Types;
import org.apache.fory.util.StringUtils;
import org.apache.fory.util.Utils;
/**
- * A decoder which decode binary into {@link TypeDef}. Global header layout follows the xlang spec
- * with an 8-bit meta size and flags at bits 8/9. See spec documentation:
+ * A decoder which decode binary into {@link TypeDef}. See spec documentation:
* docs/specification/fory_xlang_serialization_spec.md <a
* href="https://fory.apache.org/docs/specification/fory_xlang_serialization_spec">...</a>
*/
@@ -52,40 +56,88 @@ class TypeDefDecoder {
private static final Logger LOG = LoggerFactory.getLogger(TypeDefDecoder.class);
public static TypeDef decodeTypeDef(XtypeResolver resolver, MemoryBuffer inputBuffer, long id) {
+ if ((id & COMPRESS_META_FLAG) != 0) {
+ throw new DeserializationException("Compressed xlang TypeDef is not supported");
+ }
Tuple2<byte[], byte[]> decoded = decodeTypeDefBuf(inputBuffer, resolver, id);
MemoryBuffer buffer = MemoryBuffer.fromByteArray(decoded.f0);
- byte header = buffer.readByte();
- int numFields = header & SMALL_NUM_FIELDS_THRESHOLD;
- if (numFields == SMALL_NUM_FIELDS_THRESHOLD) {
- numFields += buffer.readVarUInt32Small7();
- }
+ int header = buffer.readByte() & 0xff;
+ boolean isStruct = (header & STRUCT_FLAG) != 0;
+ int numFields = 0;
ClassSpec classSpec;
- if ((header & REGISTER_BY_NAME_FLAG) != 0) {
- String namespace = readPkgName(buffer);
- String typeName = readTypeName(buffer);
- if (Utils.DEBUG_OUTPUT_ENABLED) {
- LOG.info("Decode class {} using namespace {}", typeName, namespace);
+ if (isStruct) {
+ boolean named = (header & REGISTER_BY_NAME_FLAG) != 0;
+ boolean compatible = (header & COMPATIBLE_FLAG) != 0;
+ int typeId;
+ if (named) {
+ typeId = compatible ? Types.NAMED_COMPATIBLE_STRUCT : Types.NAMED_STRUCT;
+ } else {
+ typeId = compatible ? Types.COMPATIBLE_STRUCT : Types.STRUCT;
+ }
+ numFields = header & SMALL_NUM_FIELDS_THRESHOLD;
+ if (numFields == SMALL_NUM_FIELDS_THRESHOLD) {
+ numFields += buffer.readVarUInt32Small7();
}
- TypeInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName);
- if (userTypeInfo == null) {
- classSpec = new ClassSpec(UnknownClass.UnknownStruct.class);
+ if (named) {
+ String namespace = readPkgName(buffer);
+ String typeName = readTypeName(buffer);
+ if (Utils.DEBUG_OUTPUT_ENABLED) {
+ LOG.info("Decode class {} using namespace {}", typeName, namespace);
+ }
+ TypeInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName);
+ if (userTypeInfo == null) {
+ classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, -1);
+ } else {
+ validateRegisteredTypeDefKind(userTypeInfo, typeId);
+ classSpec = new ClassSpec(userTypeInfo.getType(), typeId, userTypeInfo.getUserTypeId());
+ }
} else {
- classSpec = new ClassSpec(userTypeInfo.getType());
+ int userTypeId = buffer.readVarUInt32();
+ TypeInfo userTypeInfo = resolver.getUserTypeInfo(userTypeId);
+ if (userTypeInfo == null) {
+ classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, userTypeId);
+ } else {
+ validateRegisteredTypeDefKind(userTypeInfo, typeId);
+ classSpec = new ClassSpec(userTypeInfo.getType(), typeId, userTypeId);
+ }
}
} else {
- int typeId = buffer.readUInt8();
- int userTypeId = buffer.readVarUInt32();
- TypeInfo userTypeInfo = resolver.getUserTypeInfo(userTypeId);
- if (userTypeInfo == null) {
- classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, userTypeId);
+ if ((header & 0b0111_0000) != 0) {
+ throw new DeserializationException("Invalid TypeDef kind header");
+ }
+ int typeId = nonStructTypeId(header & 0b1111);
+ boolean named = Types.isNamedType(typeId);
+ if (named) {
+ String namespace = readPkgName(buffer);
+ String typeName = readTypeName(buffer);
+ TypeInfo userTypeInfo = resolver.getUserTypeInfo(namespace, typeName);
+ if (userTypeInfo == null) {
+ classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, -1);
+ } else {
+ validateRegisteredTypeDefKind(userTypeInfo, typeId);
+ classSpec = new ClassSpec(userTypeInfo.getType(), typeId, userTypeInfo.getUserTypeId());
+ }
} else {
- classSpec = new ClassSpec(userTypeInfo.getType(), typeId, userTypeId);
+ int userTypeId = buffer.readVarUInt32();
+ TypeInfo userTypeInfo = resolver.getUserTypeInfo(userTypeId);
+ if (userTypeInfo == null) {
+ classSpec = new ClassSpec(UnknownClass.UnknownStruct.class, typeId, userTypeId);
+ } else {
+ validateRegisteredTypeDefKind(userTypeInfo, typeId);
+ classSpec = new ClassSpec(userTypeInfo.getType(), typeId, userTypeId);
+ }
}
}
List<FieldInfo> classFields =
readFieldsInfo(buffer, resolver, classSpec.entireClassName, numFields);
- boolean hasFieldsMeta = (id & HAS_FIELDS_META_FLAG) != 0;
- TypeDef typeDef = new TypeDef(classSpec, classFields, hasFieldsMeta, id, decoded.f1);
+ if (!isStruct && !classFields.isEmpty()) {
+ throw new DeserializationException("Non-struct TypeDef cannot carry field metadata");
+ }
+ if (buffer.remaining() != 0) {
+ throw new DeserializationException("Invalid TypeDef metadata size");
+ }
+ validateParsedTypeDefHash(id, decoded.f1);
+ TypeDef typeDef = new TypeDef(classSpec, classFields, id, decoded.f1);
if (Utils.DEBUG_OUTPUT_ENABLED) {
LOG.info("[Java TypeDef DECODED] " + typeDef);
// Compute and print diff with local TypeDef
@@ -103,6 +155,49 @@ class TypeDefDecoder {
return typeDef;
}
+ private static void validateRegisteredTypeDefKind(TypeInfo userTypeInfo, int typeId) {
+ int registeredTypeId = userTypeInfo.getTypeId();
+ if (registeredTypeId != typeId && !isStructCompatibilityVariant(registeredTypeId, typeId)) {
+ throw new DeserializationException(
+ String.format(
+ "TypeDef kind %s does not match registered kind %s for %s",
+ typeId, registeredTypeId, userTypeInfo.getType()));
+ }
+ }
+
+ private static boolean isStructCompatibilityVariant(int registeredTypeId, int typeId) {
+ boolean registeredIdStruct =
+ registeredTypeId == Types.STRUCT || registeredTypeId == Types.COMPATIBLE_STRUCT;
+ boolean typeIdStruct = typeId == Types.STRUCT || typeId == Types.COMPATIBLE_STRUCT;
+ if (registeredIdStruct || typeIdStruct) {
+ return registeredIdStruct && typeIdStruct;
+ }
+ boolean registeredNamedStruct =
+ registeredTypeId == Types.NAMED_STRUCT || registeredTypeId == Types.NAMED_COMPATIBLE_STRUCT;
+ boolean typeIdNamedStruct =
+ typeId == Types.NAMED_STRUCT || typeId == Types.NAMED_COMPATIBLE_STRUCT;
+ return registeredNamedStruct && typeIdNamedStruct;
+ }
+
+ static int nonStructTypeId(int kindCode) {
+ switch (kindCode) {
+ case 0:
+ return Types.ENUM;
+ case 1:
+ return Types.NAMED_ENUM;
+ case 2:
+ return Types.EXT;
+ case 3:
+ return Types.NAMED_EXT;
+ case 4:
+ return Types.TYPED_UNION;
+ case 5:
+ return Types.NAMED_UNION;
+ default:
+ throw new DeserializationException("Unsupported TypeDef kind code " + kindCode);
+ }
+ }
+
// | header + type info + field name | ... | header + type info + field name |
private static List<FieldInfo> readFieldsInfo(
MemoryBuffer buffer, XtypeResolver resolver, String className, int numFields) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java
index d5eada0f0..4bed7c13d 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeDefEncoder.java
@@ -49,8 +49,7 @@ import org.apache.fory.util.StringUtils;
import org.apache.fory.util.Utils;
/**
- * An encoder which encode {@link TypeDef} into binary. Global header layout follows the xlang spec
- * with an 8-bit meta size and flags at bits 8/9. See spec documentation:
+ * An encoder which encode {@link TypeDef} into binary. See spec documentation:
* docs/specification/fory_xlang_serialization_spec.md <a
* href="https://fory.apache.org/docs/specification/fory_xlang_serialization_spec">...</a>
*/
@@ -115,7 +114,6 @@ class TypeDefEncoder {
new TypeDef(
new ClassSpec(type, typeInfo.getTypeId(), typeInfo.getUserTypeId()),
fieldInfos,
- true,
encodeTypeDef.getInt64(0),
typeDefBytes);
if (Utils.DEBUG_OUTPUT_ENABLED) {
@@ -125,43 +123,82 @@ class TypeDefEncoder {
}
static final int SMALL_NUM_FIELDS_THRESHOLD = 0b11111;
- static final int REGISTER_BY_NAME_FLAG = 0b100000;
+ static final int REGISTER_BY_NAME_FLAG = 0b0010_0000;
+ static final int COMPATIBLE_FLAG = 0b0100_0000;
+ static final int STRUCT_FLAG = 0b1000_0000;
static final int FIELD_NAME_SIZE_THRESHOLD = 0b1111;
// see spec documentation: docs/specification/xlang_serialization_spec.md
// https://fory.apache.org/docs/specification/fory_xlang_serialization_spec
static MemoryBuffer encodeTypeDef(XtypeResolver resolver, Class<?> type, List<FieldInfo> fields) {
TypeInfo typeInfo = resolver.getTypeInfo(type);
+ int typeId = typeInfo.getTypeId();
+ boolean isStruct = Types.isStructType(typeId);
+ Preconditions.checkArgument(
+ isStruct || fields.isEmpty(), "Non-struct TypeDef %s cannot carry field metadata", typeId);
MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(128);
buffer.writeByte(-1); // placeholder for header, update later
- int currentClassHeader = fields.size();
- if (fields.size() >= SMALL_NUM_FIELDS_THRESHOLD) {
- currentClassHeader = SMALL_NUM_FIELDS_THRESHOLD;
- buffer.writeVarUInt32(fields.size() - SMALL_NUM_FIELDS_THRESHOLD);
- }
- if (resolver.isRegisteredById(type)) {
- buffer.writeUInt8(typeInfo.getTypeId());
- Preconditions.checkArgument(
- typeInfo.getUserTypeId() != -1,
- "User type id is required for typeId %s",
- typeInfo.getTypeId());
- buffer.writeVarUInt32(typeInfo.getUserTypeId());
+ if (isStruct) {
+ int fieldCount = fields.size();
+ int currentClassHeader = STRUCT_FLAG | Math.min(fieldCount, SMALL_NUM_FIELDS_THRESHOLD);
+ if (typeId == Types.COMPATIBLE_STRUCT || typeId == Types.NAMED_COMPATIBLE_STRUCT) {
+ currentClassHeader |= COMPATIBLE_FLAG;
+ }
+ if (fieldCount >= SMALL_NUM_FIELDS_THRESHOLD) {
+ buffer.writeVarUInt32(fieldCount - SMALL_NUM_FIELDS_THRESHOLD);
+ }
+ if (resolver.isRegisteredById(type)) {
+ Preconditions.checkArgument(
+ typeInfo.getUserTypeId() != -1,
+ "User type id is required for typeId %s",
+ typeInfo.getTypeId());
+ buffer.writeVarUInt32(typeInfo.getUserTypeId());
+ } else {
+ Preconditions.checkArgument(resolver.isRegisteredByName(type));
+ currentClassHeader |= REGISTER_BY_NAME_FLAG;
+ String ns = typeInfo.decodeNamespace();
+ String typename = typeInfo.decodeTypeName();
+ writePkgName(buffer, ns);
+ writeTypeName(buffer, typename);
+ }
+ buffer.putByte(0, currentClassHeader);
+ writeFieldsInfo(resolver, buffer, fields);
} else {
- Preconditions.checkArgument(resolver.isRegisteredByName(type));
- currentClassHeader |= REGISTER_BY_NAME_FLAG;
- String ns = typeInfo.decodeNamespace();
- String typename = typeInfo.decodeTypeName();
- writePkgName(buffer, ns);
- writeTypeName(buffer, typename);
+ buffer.putByte(0, nonStructKindCode(typeId));
+ if (resolver.isRegisteredById(type)) {
+ Preconditions.checkArgument(
+ typeInfo.getUserTypeId() != -1,
+ "User type id is required for typeId %s",
+ typeInfo.getTypeId());
+ buffer.writeVarUInt32(typeInfo.getUserTypeId());
+ } else {
+ Preconditions.checkArgument(resolver.isRegisteredByName(type));
+ String ns = typeInfo.decodeNamespace();
+ String typename = typeInfo.decodeTypeName();
+ writePkgName(buffer, ns);
+ writeTypeName(buffer, typename);
+ }
}
- buffer.putByte(0, currentClassHeader);
- writeFieldsInfo(resolver, buffer, fields);
+ return prependHeader(buffer, false);
+ }
- // Temporary xlang behavior: always write TypeMeta uncompressed.
- // Some runtimes still don't support TypeMeta decompression, so we must avoid emitting
- // compressed xlang TypeMeta until all xlang implementations support decompress.
- // Note: native mode is unchanged and still uses NativeTypeDefEncoder compression flow.
- return prependHeader(buffer, false, !fields.isEmpty());
+ static int nonStructKindCode(int typeId) {
+ switch (typeId) {
+ case Types.ENUM:
+ return 0;
+ case Types.NAMED_ENUM:
+ return 1;
+ case Types.EXT:
+ return 2;
+ case Types.NAMED_EXT:
+ return 3;
+ case Types.TYPED_UNION:
+ return 4;
+ case Types.NAMED_UNION:
+ return 5;
+ default:
+ throw new IllegalArgumentException("Unsupported TypeDef kind " + typeId);
+ }
}
static Map<String, FieldInfo> getClassFields(Class<?> type, List<FieldInfo> fieldsInfo) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/meta/TypeEqualMetaCompressor.java b/java/fory-core/src/main/java/org/apache/fory/meta/TypeEqualMetaCompressor.java
index 1c21296ce..34eb2fe2d 100644
--- a/java/fory-core/src/main/java/org/apache/fory/meta/TypeEqualMetaCompressor.java
+++ b/java/fory-core/src/main/java/org/apache/fory/meta/TypeEqualMetaCompressor.java
@@ -52,6 +52,11 @@ class TypeEqualMetaCompressor implements MetaCompressor {
return compressor.decompress(data, offset, size);
}
+ @Override
+ public byte[] decompress(byte[] data, int offset, int size, int maxOutputSize) {
+ return compressor.decompress(data, offset, size, maxOutputSize);
+ }
+
@Override
public boolean equals(Object obj) {
if (obj == null || obj.getClass() != getClass()) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
index c1d7493ff..6a69503ea 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/ClassResolver.java
@@ -41,6 +41,7 @@ import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
+import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.EnumSet;
@@ -101,6 +102,7 @@ import org.apache.fory.memory.Platform;
import org.apache.fory.meta.ClassSpec;
import org.apache.fory.meta.EncodedMetaString;
import org.apache.fory.meta.Encoders;
+import org.apache.fory.meta.NativeTypeDefEncoder;
import org.apache.fory.meta.TypeDef;
import org.apache.fory.reflect.ObjectCreators;
import org.apache.fory.reflect.ReflectionUtils;
@@ -537,7 +539,7 @@ public class ClassResolver extends TypeResolver {
buildUnregisteredTypeId(cls, existingInfo == null ? null : existingInfo.serializer);
TypeInfo typeInfo = new TypeInfo(cls, nsBytes, nameBytes, null, typeId, -1);
classInfoMap.put(cls, typeInfo);
- compositeNameBytes2TypeInfo.put(new TypeNameBytes(nsBytes.hash, nameBytes.hash), typeInfo);
+ compositeNameBytes2TypeInfo.put(new TypeNameBytes(nsBytes, nameBytes), typeInfo);
extRegistry.registeredClasses.put(fullname, cls);
registerGraalvmClass(cls);
}
@@ -583,7 +585,7 @@ public class ClassResolver extends TypeResolver {
TypeInfo typeInfo = new TypeInfo(cls, nsBytes, nameBytes, serializer, typeId, -1);
typeInfo.setSerializer(this, serializer);
classInfoMap.put(cls, typeInfo);
- compositeNameBytes2TypeInfo.put(new TypeNameBytes(nsBytes.hash, nameBytes.hash), typeInfo);
+ compositeNameBytes2TypeInfo.put(new TypeNameBytes(nsBytes, nameBytes), typeInfo);
extRegistry.registeredClasses.put(fullname, cls);
registerGraalvmClass(cls);
}
@@ -869,17 +871,92 @@ public class ClassResolver extends TypeResolver {
}
return typeInfo.typeId;
}
- int typeId = buildUnregisteredTypeId(cls, null);
+ int typeId = usesNonStructTypeDef(cls) ? Types.NAMED_EXT : buildUnregisteredTypeId(cls, null);
typeInfo = new TypeInfo(this, cls, null, typeId, INVALID_USER_TYPE_ID);
classInfoMap.put(cls, typeInfo);
if (typeInfo.namespace != null && typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo);
}
return typeId;
}
+ public int getTypeDefRootTypeId(Class<?> cls, boolean hasFieldMetadata) {
+ if (hasFieldMetadata) {
+ // Preserve the normal TypeInfo/name cache so locally generated or dynamically registered
+ // classes can be resolved when the TypeDef is decoded by the same resolver.
+ getTypeIdForTypeDef(cls);
+ return getFieldMetadataTypeIdForTypeDef(cls);
+ }
+ TypeInfo typeInfo = classInfoMap.get(cls);
+ if (typeInfo != null) {
+ return normalizeTypeDefRootTypeId(cls, typeInfo.typeId);
+ }
+ Integer classId = extRegistry.registeredClassIdMap.get(cls);
+ if (classId != null) {
+ typeInfo = classInfoMap.get(cls);
+ if (typeInfo == null) {
+ typeInfo = getTypeInfo(cls);
+ }
+ return normalizeTypeDefRootTypeId(cls, typeInfo.typeId);
+ }
+ return usesNonStructTypeDef(cls) ? Types.NAMED_EXT : buildUnregisteredTypeId(cls, null);
+ }
+
+ private int getFieldMetadataTypeIdForTypeDef(Class<?> cls) {
+ Integer classId = extRegistry.registeredClassIdMap.get(cls);
+ if (classId != null && !isInternalRegisteredClassId(cls, classId)) {
+ return buildUserTypeId(cls, null);
+ }
+ return super.buildUnregisteredTypeId(cls, null);
+ }
+
+ private int normalizeTypeDefRootTypeId(Class<?> cls, int typeId) {
+ if (usesNonStructTypeDef(cls)) {
+ // Placeholder TypeInfo can be created before the natural serializer is installed.
+ // The TypeDef root kind must still select the non-struct serializer family.
+ return Types.isExtType(typeId) ? typeId : Types.NAMED_EXT;
+ }
+ if (isSupportedTypeDefTypeId(typeId)) {
+ return typeId;
+ }
+ return buildUnregisteredTypeId(cls, null);
+ }
+
+ private static boolean isSupportedTypeDefTypeId(int typeId) {
+ switch (typeId) {
+ case Types.ENUM:
+ case Types.NAMED_ENUM:
+ case Types.STRUCT:
+ case Types.COMPATIBLE_STRUCT:
+ case Types.NAMED_STRUCT:
+ case Types.NAMED_COMPATIBLE_STRUCT:
+ case Types.EXT:
+ case Types.NAMED_EXT:
+ case Types.TYPED_UNION:
+ case Types.NAMED_UNION:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ private boolean usesNonStructTypeDef(Class<?> cls) {
+ return !cls.isEnum()
+ && (cls.isArray()
+ || isCollection(cls)
+ || isMap(cls)
+ || Externalizable.class.isAssignableFrom(cls)
+ || requireJavaSerialization(cls)
+ || useReplaceResolveSerializer(cls)
+ || Functions.isLambda(cls)
+ || (config.isScalaOptimizationEnabled() && ReflectionUtils.isScalaSingletonObject(cls))
+ || Calendar.class.isAssignableFrom(cls)
+ || ZoneId.class.isAssignableFrom(cls)
+ || TimeZone.class.isAssignableFrom(cls)
+ || ByteBuffer.class.isAssignableFrom(cls));
+ }
+
/**
* Compute the user type id used in TypeDef without forcing serializer creation. Returns -1 when
* the class isn't registered by numeric id.
@@ -1084,8 +1161,7 @@ public class ClassResolver extends TypeResolver {
typeInfo = sharedTypeInfo;
updateTypeInfo(type, typeInfo);
if (typeInfo.namespace != null && typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo);
}
if (typeInfoCache.type == type) {
@@ -1235,8 +1311,7 @@ public class ClassResolver extends TypeResolver {
// readTypeInfo can find the TypeInfo by name bytes during deserialization.
// This is important for dynamically created classes that can't be loaded by name.
if (typeInfo.namespace != null && typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeNameBytes2TypeInfo.put(typeNameBytes, typeInfo);
}
}
@@ -1717,7 +1792,6 @@ public class ClassResolver extends TypeResolver {
getGraalvmClassRegistry()
.putDeserializerClass(
typeDef.getId(), getMetaSharedDeserializerClassForGraalvmBuild(cls, typeDef));
- extRegistry.typeInfoByTypeDefId.remove(typeDef.getId());
}
typeInfoCache = NIL_TYPE_INFO;
if (RecordUtils.isRecord(cls)) {
@@ -1818,15 +1892,15 @@ public class ClassResolver extends TypeResolver {
Preconditions.checkArgument(
serializerClass != UnknownClassSerializers.UnknownStructSerializer.class);
if (needToWriteTypeDef(serializerClass)) {
- typeDef =
- cacheTypeDef(
- typeDefMap.computeIfAbsent(typeInfo.type, cls -> TypeDef.buildTypeDef(this, cls)));
+ typeDef = typeDefMap.computeIfAbsent(typeInfo.type, cls -> TypeDef.buildTypeDef(this, cls));
} else {
// Some type will use other serializers such MapSerializer and so on.
typeDef =
- cacheTypeDef(
- typeDefMap.computeIfAbsent(
- typeInfo.type, cls -> TypeDef.buildTypeDef(this, cls, new ArrayList<>(), false)));
+ typeDefMap.computeIfAbsent(
+ typeInfo.type,
+ cls ->
+ NativeTypeDefEncoder.buildTypeDefWithFieldInfos(
+ this, cls, Collections.emptyList()));
}
typeInfo.typeDef = typeDef;
return typeDef;
@@ -1933,7 +2007,7 @@ public class ClassResolver extends TypeResolver {
@Override
protected TypeInfo loadBytesToTypeInfo(
EncodedMetaString packageBytes, EncodedMetaString simpleClassNameBytes) {
- TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hash, simpleClassNameBytes.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes, simpleClassNameBytes);
TypeInfo typeInfo = compositeNameBytes2TypeInfo.get(typeNameBytes);
if (typeInfo == null) {
typeInfo = populateBytesToTypeInfo(typeNameBytes, packageBytes, simpleClassNameBytes);
@@ -1955,8 +2029,7 @@ public class ClassResolver extends TypeResolver {
TypeInfo newTypeInfo = getTypeInfo(typeInfo.type);
// Update the cache with the correct TypeInfo that has a serializer
if (typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeNameBytes2TypeInfo.put(typeNameBytes, newTypeInfo);
}
return newTypeInfo;
@@ -1994,8 +2067,7 @@ public class ClassResolver extends TypeResolver {
String typeName = ReflectionUtils.getClassNameWithoutPackage(className);
EncodedMetaString pkgBytes = sharedRegistry.getPackageEncodedMetaString(pkg);
EncodedMetaString typeBytes = sharedRegistry.getTypeNameEncodedMetaString(typeName);
- TypeInfo cachedInfo =
- compositeNameBytes2TypeInfo.get(new TypeNameBytes(pkgBytes.hash, typeBytes.hash));
+ TypeInfo cachedInfo = compositeNameBytes2TypeInfo.get(new TypeNameBytes(pkgBytes, typeBytes));
if (cachedInfo != null) {
return cachedInfo.type;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java
index 8e6b50484..5dec2d5ff 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/SharedRegistry.java
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Objects;
import java.util.SortedMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.fory.annotation.Internal;
import org.apache.fory.codegen.CodeGenerator;
import org.apache.fory.collection.BiMap;
@@ -54,6 +55,7 @@ import org.apache.fory.util.GraalvmSupport;
public final class SharedRegistry {
private static final int MAX_CACHED_ENCODED_META_STRINGS = 32768;
private static final int MAX_CACHED_ENCODED_META_STRING_LENGTH = 2048;
+ private static final int MAX_CACHED_TYPE_DEFS = 8192;
final ConcurrentIdentityMap<Class<?>, TypeDef> typeDefMap = new ConcurrentIdentityMap<>();
final ConcurrentIdentityMap<Class<?>, TypeDef> currentLayerTypeDef =
@@ -80,6 +82,7 @@ public final class SharedRegistry {
final ConcurrentIdentityMap<Class<?>, Serializer<?>> registeredSerializerCache =
new ConcurrentIdentityMap<>();
private final Object metaStringCacheLock = new Object();
+ private final AtomicInteger cachedTypeDefCount = new AtomicInteger();
volatile IdentityHashMap<Class<?>, Integer> registeredClassIdMap;
volatile BiMap<String, Class<?>> registeredClasses;
@@ -125,6 +128,43 @@ public final class SharedRegistry {
return existing == null ? typeInfo : existing;
}
+ TypeDef getOrCreateTypeDef(TypeDef typeDef) {
+ long id = typeDef.getId();
+ TypeDef existing = typeDefById.get(id);
+ if (existing != null) {
+ return existing;
+ }
+ if (!reserveTypeDefCacheSlot()) {
+ return typeDef;
+ }
+ existing = typeDefById.putIfAbsent(id, typeDef);
+ if (existing != null) {
+ cachedTypeDefCount.decrementAndGet();
+ return existing;
+ }
+ return typeDef;
+ }
+
+ private boolean reserveTypeDefCacheSlot() {
+ while (true) {
+ int count = cachedTypeDefCount.get();
+ int mapSize = typeDefById.size();
+ if (mapSize > count) {
+ if (cachedTypeDefCount.compareAndSet(count, mapSize)) {
+ count = mapSize;
+ } else {
+ continue;
+ }
+ }
+ if (count >= MAX_CACHED_TYPE_DEFS) {
+ return false;
+ }
+ if (cachedTypeDefCount.compareAndSet(count, count + 1)) {
+ return true;
+ }
+ }
+ }
+
EncodedMetaString getPackageEncodedMetaString(String string) {
return getEncodedMetaString(
string,
@@ -210,11 +250,6 @@ public final class SharedRegistry {
return encodedMetaString.bytes.length <= MAX_CACHED_ENCODED_META_STRING_LENGTH;
}
- TypeDef getOrCreateTypeDef(TypeDef typeDef) {
- TypeDef existingTypeDef = typeDefById.putIfAbsent(typeDef.getId(), typeDef);
- return existingTypeDef == null ? typeDef : existingTypeDef;
- }
-
List<Descriptor> getOrCreateFieldDescriptors(
Class<?> type, boolean searchParent, java.util.function.Supplier<List<Descriptor>> factory) {
if (GraalvmSupport.isGraalBuildtime()) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeNameBytes.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeNameBytes.java
index 20fe81482..f0ae54e53 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeNameBytes.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeNameBytes.java
@@ -19,20 +19,30 @@
package org.apache.fory.resolver;
+import java.util.Arrays;
+import org.apache.fory.meta.EncodedMetaString;
+
class TypeNameBytes {
private final long packageHash;
private final long classNameHash;
+ private final byte[] packageBytes;
+ private final byte[] classNameBytes;
- TypeNameBytes(long packageHash, long classNameHash) {
- this.packageHash = packageHash;
- this.classNameHash = classNameHash;
+ TypeNameBytes(EncodedMetaString packageBytes, EncodedMetaString classNameBytes) {
+ this.packageHash = packageBytes.hash;
+ this.classNameHash = classNameBytes.hash;
+ this.packageBytes = packageBytes.bytes;
+ this.classNameBytes = classNameBytes.bytes;
}
@Override
public boolean equals(Object o) {
// ClassNameBytes is used internally, skip
TypeNameBytes that = (TypeNameBytes) o;
- return packageHash == that.packageHash && classNameHash == that.classNameHash;
+ return packageHash == that.packageHash
+ && classNameHash == that.classNameHash
+ && Arrays.equals(packageBytes, that.packageBytes)
+ && Arrays.equals(classNameBytes, that.classNameBytes);
}
@Override
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java
index 37cbe1e9b..c5856d6d2 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/TypeResolver.java
@@ -64,6 +64,7 @@ import org.apache.fory.context.MetaWriteContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
import org.apache.fory.exception.ForyException;
+import org.apache.fory.exception.InsecureException;
import org.apache.fory.exception.SerializerUnregisteredException;
import org.apache.fory.logging.Logger;
import org.apache.fory.logging.LoggerFactory;
@@ -116,8 +117,21 @@ public abstract class TypeResolver {
static final int INTERNAL_NATIVE_ID_LIMIT = 250;
private static final GenericType OBJECT_GENERIC_TYPE = GenericType.build(Object.class);
private static final float TYPE_ID_MAP_LOAD_FACTOR = 0.5f;
+ private static final int MAX_CACHED_TYPE_DEFS = 8192;
static final long MAX_USER_TYPE_ID = 0xffff_fffEL;
+ private static final class TransformedTypeInfo {
+ final Class<?> readClass;
+ final long typeDefId;
+ final TypeInfo typeInfo;
+
+ TransformedTypeInfo(Class<?> readClass, long typeDefId, TypeInfo typeInfo) {
+ this.readClass = readClass;
+ this.typeDefId = typeDefId;
+ this.typeInfo = typeInfo;
+ }
+ }
+
final Config config;
final boolean metaContextShareEnabled;
final SharedRegistry sharedRegistry;
@@ -729,7 +743,7 @@ public abstract class TypeResolver {
/**
* Read class info from bytes with cache optimization. Uses the cached namespace and type name
- * bytes to avoid map lookups when the class is the same as the cached one (hash comparison).
+ * bytes to avoid map lookups when the class is the same as the cached one.
*/
protected final TypeInfo readTypeInfoFromBytes(
ReadContext readContext, TypeInfo typeInfoCache, int header) {
@@ -746,9 +760,9 @@ public abstract class TypeResolver {
assert packageNameBytesCache != null;
simpleClassNameBytes = metaStringReader.readMetaString(buffer, typeNameBytesCache);
- // Fast path: if hashes match, return cached TypeInfo (already has serializer)
- if (typeNameBytesCache.hash == simpleClassNameBytes.hash
- && packageNameBytesCache.hash == namespaceBytes.hash) {
+ // MetaStringReader returns the provided cache object only when the wire identity matches. For
+ // big meta strings, body-hash validation happens before the entry is first cached.
+ if (typeNameBytesCache == simpleClassNameBytes && packageNameBytesCache == namespaceBytes) {
return typeInfoCache;
}
} else {
@@ -775,9 +789,11 @@ public abstract class TypeResolver {
TypeInfo typeInfo;
if (isRef) {
// Reference to previously read type in this stream
- typeInfo = metaReadContext.readTypeInfos.get(index);
+ typeInfo = getMetaReadTypeInfo(metaReadContext, index);
} else {
- // New type in stream - but may already be known from registry
+ // New type in stream, with optimized reuse by validated TypeDef header. A header-cache
+ // hit intentionally skips the body without rehashing: entries are published only after the
+ // TypeDef body has parsed successfully and matched the 52-bit body hash.
long id = buffer.readInt64();
typeInfo = extRegistry.typeInfoByTypeDefId.get(id);
if (typeInfo != null) {
@@ -807,21 +823,38 @@ public abstract class TypeResolver {
return typeInfo;
}
+ private static TypeInfo getMetaReadTypeInfo(MetaReadContext metaReadContext, int index) {
+ if (index < 0 || index >= metaReadContext.readTypeInfos.size) {
+ throw new ForyException("Invalid class metadata reference id " + index);
+ }
+ TypeInfo typeInfo = metaReadContext.readTypeInfos.get(index);
+ if (typeInfo == null) {
+ throw new ForyException("Invalid class metadata reference id " + index);
+ }
+ return typeInfo;
+ }
+
private TypeInfo getTargetTypeInfo(TypeInfo typeInfo, Class<?> targetClass) {
- Tuple2<Class<?>, TypeInfo>[] infos = extRegistry.transformedTypeInfo.get(targetClass);
+ TransformedTypeInfo[] infos = extRegistry.transformedTypeInfo.get(targetClass);
Class<?> readClass = typeInfo.getType();
+ long typeDefId = transformCacheTypeDefId(typeInfo);
if (infos != null) {
// It's ok to use loop here since most of case the array size will be 1.
- for (Tuple2<Class<?>, TypeInfo> info : infos) {
- if (info.f0 == readClass) {
- return info.f1;
+ for (TransformedTypeInfo info : infos) {
+ if (info.readClass == readClass && info.typeDefId == typeDefId) {
+ return info.typeInfo;
}
}
}
- return transformTypeInfo(typeInfo, targetClass);
+ return transformTypeInfo(typeInfo, targetClass, typeDefId);
+ }
+
+ private static long transformCacheTypeDefId(TypeInfo typeInfo) {
+ TypeDef typeDef = typeInfo.getTypeDef();
+ return typeDef == null ? 0 : typeDef.getId();
}
- private TypeInfo transformTypeInfo(TypeInfo typeInfo, Class<?> targetClass) {
+ private TypeInfo transformTypeInfo(TypeInfo typeInfo, Class<?> targetClass, long typeDefId) {
Class<?> readClass = typeInfo.getType();
TypeInfo newTypeInfo;
if (targetClass.isAssignableFrom(readClass)) {
@@ -832,14 +865,16 @@ public abstract class TypeResolver {
getMetaSharedTypeInfo(
typeInfo.typeDef.replaceRootClassTo(this, targetClass), targetClass);
}
- Tuple2<Class<?>, TypeInfo>[] infos = extRegistry.transformedTypeInfo.get(targetClass);
+ TransformedTypeInfo[] infos = extRegistry.transformedTypeInfo.get(targetClass);
int size = infos == null ? 0 : infos.length;
- @SuppressWarnings("unchecked")
- Tuple2<Class<?>, TypeInfo>[] newInfos = (Tuple2<Class<?>, TypeInfo>[]) new Tuple2[size + 1];
+ if (size >= MAX_CACHED_TYPE_DEFS) {
+ return newTypeInfo;
+ }
+ TransformedTypeInfo[] newInfos = new TransformedTypeInfo[size + 1];
if (size > 0) {
System.arraycopy(infos, 0, newInfos, 0, size);
}
- newInfos[size] = Tuple2.of(readClass, newTypeInfo);
+ newInfos[size] = new TransformedTypeInfo(readClass, typeDefId, newTypeInfo);
extRegistry.transformedTypeInfo.put(targetClass, newInfos);
return newTypeInfo;
}
@@ -921,10 +956,7 @@ public abstract class TypeResolver {
return typeInfo;
}
Class<?> cls = loadClass(typeDef.getClassSpec());
- // For nonexistent classes, always create a new TypeInfo with the correct typeDef,
- // even if the typeDef has no fields meta. This ensures the UnknownStructSerializer
- // has access to the typeDef for proper deserialization.
- if (!typeDef.hasFieldsMeta()
+ if (!typeDef.isStructSchemaKind()
&& !UnknownClass.class.isAssignableFrom(TypeUtils.getComponentIfArray(cls))) {
typeInfo = getTypeInfo(cls);
} else if (ClassResolver.useReplaceResolveSerializer(cls)) {
@@ -934,7 +966,9 @@ public abstract class TypeResolver {
} else {
typeInfo = getMetaSharedTypeInfo(typeDef, cls);
}
- extRegistry.typeInfoByTypeDefId.put(typeDef.getId(), typeInfo);
+ if (extRegistry.typeInfoByTypeDefId.size < MAX_CACHED_TYPE_DEFS) {
+ extRegistry.typeInfoByTypeDefId.put(typeDef.getId(), typeInfo);
+ }
return typeInfo;
}
@@ -1080,7 +1114,10 @@ public abstract class TypeResolver {
final Class<?> loadClass(
String className, boolean isEnum, int arrayDims, boolean deserializeUnknownClass) {
- extRegistry.typeChecker.checkType(this, className);
+ if (!extRegistry.typeChecker.checkType(this, className)) {
+ throw new InsecureException(
+ String.format("Class %s is forbidden for serialization.", className));
+ }
Class<?> cls = extRegistry.registeredClasses.get(className);
if (cls != null) {
return cls;
@@ -1807,15 +1844,12 @@ public abstract class TypeResolver {
int userIdGenerator = 0;
SerializerFactory serializerFactory;
final LongMap<TypeInfo> typeInfoByTypeDefId = new LongMap<>(2, 0.5f);
-
// cache absTypeInfo, support customized serializer for abstract or interface.
// IdentityHashMap is more memory efficient than fory IdentityMap, and this is not in hotpath
// for query
final IdentityHashMap<Class<?>, TypeInfo> abstractTypeInfo = new IdentityHashMap<>();
- // Tuple2<Class, Class>: Tuple2<From Class, To Class>
- final IdentityHashMap<Class<?>, Tuple2<Class<?>, TypeInfo>[]> transformedTypeInfo =
+ final IdentityHashMap<Class<?>, TransformedTypeInfo[]> transformedTypeInfo =
new IdentityHashMap<>();
-
// avoid potential recursive call for seq codec generation.
// ex. A->field1: B, B.field1: A
final Set<Class<?>> getClassCtx = new HashSet<>();
diff --git a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
index 3f1ee75fe..39e70a543 100644
--- a/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
+++ b/java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java
@@ -334,8 +334,7 @@ public class XtypeResolver extends TypeResolver {
typeInfo.setSerializer(this, serializer);
extRegistry.registeredClasses.put(qualifiedName, type);
if (typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo);
}
registerGraalvmClass(type);
@@ -481,8 +480,7 @@ public class XtypeResolver extends TypeResolver {
}
updateTypeInfo(type, typeInfo);
if (typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeClassNameBytes2TypeInfo.put(typeNameBytes, typeInfo);
}
}
@@ -740,8 +738,7 @@ public class XtypeResolver extends TypeResolver {
public TypeInfo getUserTypeInfo(String namespace, String typeName) {
EncodedMetaString namespaceBytes = sharedRegistry.getPackageEncodedMetaString(namespace);
EncodedMetaString typeNameBytes = sharedRegistry.getTypeNameEncodedMetaString(typeName);
- return compositeClassNameBytes2TypeInfo.get(
- new TypeNameBytes(namespaceBytes.hash, typeNameBytes.hash));
+ return compositeClassNameBytes2TypeInfo.get(new TypeNameBytes(namespaceBytes, typeNameBytes));
}
public TypeInfo getUserTypeInfo(int userTypeId) {
@@ -1113,8 +1110,7 @@ public class XtypeResolver extends TypeResolver {
@Override
protected TypeDef buildTypeDef(TypeInfo typeInfo) {
TypeDef typeDef =
- cacheTypeDef(
- typeDefMap.computeIfAbsent(typeInfo.type, cls -> TypeDef.buildTypeDef(this, cls)));
+ typeDefMap.computeIfAbsent(typeInfo.type, cls -> TypeDef.buildTypeDef(this, cls));
typeInfo.typeDef = typeDef;
return typeDef;
}
@@ -1196,8 +1192,7 @@ public class XtypeResolver extends TypeResolver {
TypeInfo newTypeInfo = getTypeInfo(typeInfo.type);
// Update the cache with the correct TypeInfo that has a serializer
if (typeInfo.typeName != null) {
- TypeNameBytes typeNameBytes =
- new TypeNameBytes(typeInfo.namespace.hash, typeInfo.typeName.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(typeInfo.namespace, typeInfo.typeName);
compositeClassNameBytes2TypeInfo.put(typeNameBytes, newTypeInfo);
}
return newTypeInfo;
@@ -1207,7 +1202,7 @@ public class XtypeResolver extends TypeResolver {
private TypeInfo loadBytesToTypeInfoWithTypeId(
int internalTypeId, EncodedMetaString packageBytes, EncodedMetaString simpleClassNameBytes) {
- TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes.hash, simpleClassNameBytes.hash);
+ TypeNameBytes typeNameBytes = new TypeNameBytes(packageBytes, simpleClassNameBytes);
TypeInfo typeInfo = compositeClassNameBytes2TypeInfo.get(typeNameBytes);
if (typeInfo == null) {
typeInfo =
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java
index b5bb42eb2..24a6a251a 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ArraySerializers.java
@@ -24,6 +24,7 @@ import org.apache.fory.Fory;
import org.apache.fory.context.CopyContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.resolver.ClassResolver;
import org.apache.fory.resolver.RefMode;
@@ -46,6 +47,19 @@ import org.apache.fory.util.Preconditions;
public final class ArraySerializers {
private ArraySerializers() {}
+ private static void throwObjectArraySizeLimitExceeded(int size, int maxCollectionSize) {
+ throw new DeserializationException(
+ "Object array size " + size + " exceeds max collection size " + maxCollectionSize);
+ }
+
+ private static void throwInvalidObjectArraySize(int size, int maxCollectionSize) {
+ if (size < 0) {
+ throw new DeserializationException("Object array size must be non-negative: " + size);
+ } else {
+ throwObjectArraySizeLimitExceeded(size, maxCollectionSize);
+ }
+ }
+
/**
* Returns the object-array serializer for {@code cls}.
*
@@ -85,6 +99,7 @@ public final class ArraySerializers {
public static final class ObjectArraySerializer extends Serializer<Object[]> {
private final TypeResolver typeResolver;
private final TypeInfoHolder elementTypeInfoHolder;
+ private final int maxCollectionSize;
public ObjectArraySerializer(TypeResolver typeResolver, Class<?> cls) {
super(typeResolver.getConfig(), (Class) cls);
@@ -94,6 +109,7 @@ public final class ArraySerializers {
}
Preconditions.checkArgument(cls.isArray() && !cls.getComponentType().isPrimitive());
elementTypeInfoHolder = typeResolver.nilTypeInfoHolder();
+ maxCollectionSize = typeResolver.getConfig().maxCollectionSize();
}
@Override
@@ -125,6 +141,11 @@ public final class ArraySerializers {
public Object[] read(ReadContext readContext) {
MemoryBuffer buffer = readContext.getBuffer();
int numElements = buffer.readVarUInt32Small7();
+ // Keep this as direct primitive branches. Object-array reads allocate immediately; using
+ // Preconditions.checkArgument here would add helper/varargs overhead on the valid path.
+ if (numElements < 0 || numElements > maxCollectionSize) {
+ throwInvalidObjectArraySize(numElements, maxCollectionSize);
+ }
Object[] value = newArray(numElements);
readContext.reference(value);
if (numElements != 0) {
@@ -158,6 +179,7 @@ public final class ArraySerializers {
private final Class<?> componentType;
private final Serializer elementSerializer;
private final TypeInfoHolder elementTypeInfoHolder;
+ private final int maxCollectionSize;
SameTypeObjectArraySerializer(
TypeResolver typeResolver, Class<?> arrayType, Class<?> componentType) {
@@ -169,6 +191,7 @@ public final class ArraySerializers {
}
elementSerializer = typeResolver.getSerializer(componentType);
elementTypeInfoHolder = typeResolver.nilTypeInfoHolder();
+ maxCollectionSize = typeResolver.getConfig().maxCollectionSize();
}
@Override
@@ -204,6 +227,11 @@ public final class ArraySerializers {
public Object[] read(ReadContext readContext) {
MemoryBuffer buffer = readContext.getBuffer();
int numElements = buffer.readVarUInt32Small7();
+ // Keep this as direct primitive branches. Object-array reads allocate immediately; using
+ // Preconditions.checkArgument here would add helper/varargs overhead on the valid path.
+ if (numElements < 0 || numElements > maxCollectionSize) {
+ throwInvalidObjectArraySize(numElements, maxCollectionSize);
+ }
Object[] value = newArray(numElements);
readContext.reference(value);
if (numElements != 0) {
@@ -616,6 +644,7 @@ public final class ArraySerializers {
private final String className;
private final TypeResolver typeResolver;
private final TypeInfoHolder elementTypeInfoHolder;
+ private final int maxCollectionSize;
public UnknownArraySerializer(TypeResolver typeResolver, Class<?> cls) {
this(typeResolver, "Unknown", cls);
@@ -627,6 +656,7 @@ public final class ArraySerializers {
this.className = className;
this.typeResolver = typeResolver;
elementTypeInfoHolder = typeResolver.nilTypeInfoHolder();
+ maxCollectionSize = typeResolver.getConfig().maxCollectionSize();
}
@Override
@@ -639,6 +669,11 @@ public final class ArraySerializers {
public Object[] read(ReadContext readContext) {
MemoryBuffer buffer = readContext.getBuffer();
int numElements = buffer.readVarUInt32Small7();
+ // Keep this as direct primitive branches. Object-array reads allocate immediately; using
+ // Preconditions.checkArgument here would add helper/varargs overhead on the valid path.
+ if (numElements < 0 || numElements > maxCollectionSize) {
+ throwInvalidObjectArraySize(numElements, maxCollectionSize);
+ }
Object[] value = newArray(numElements);
readContext.reference(value);
if (numElements != 0) {
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/BigIntegerSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/BigIntegerSerializer.java
index f5641d770..6378e5e72 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/BigIntegerSerializer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/BigIntegerSerializer.java
@@ -23,16 +23,19 @@ import java.math.BigInteger;
import org.apache.fory.config.Config;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.MemoryBuffer;
/** Serializer for {@link BigInteger} in native and xlang modes. */
public final class BigIntegerSerializer extends ImmutableSerializer<BigInteger>
implements Shareable {
private final boolean xlang;
+ private final int maxBinarySize;
public BigIntegerSerializer(Config config) {
super(config, BigInteger.class);
xlang = config.isXlang();
+ maxBinarySize = config.maxBinarySize();
}
@Override
@@ -62,6 +65,8 @@ public final class BigIntegerSerializer extends ImmutableSerializer<BigInteger>
private BigInteger readNative(ReadContext readContext) {
MemoryBuffer buffer = readContext.getBuffer();
int len = buffer.readVarUInt32Small7();
+ checkBinaryPayloadLength(len, maxBinarySize);
+ buffer.checkReadableBytes(len);
byte[] bytes = buffer.readBytes(len);
return new BigInteger(bytes);
}
@@ -71,6 +76,16 @@ public final class BigIntegerSerializer extends ImmutableSerializer<BigInteger>
}
private BigInteger readXlang(ReadContext readContext) {
- return DecimalSerializer.readXlangBigInteger(readContext.getBuffer());
+ return DecimalSerializer.readXlangBigInteger(readContext.getBuffer(), maxBinarySize);
+ }
+
+ private static void checkBinaryPayloadLength(int len, int maxBinarySize) {
+ if (len <= 0) {
+ throw new DeserializationException("BigInteger payload length must be positive: " + len);
+ }
+ if (len > maxBinarySize) {
+ throw new DeserializationException(
+ "BigInteger payload length " + len + " exceeds max binary size " + maxBinarySize);
+ }
}
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/BufferSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/BufferSerializers.java
index d5d52a4ce..74619ff9f 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/BufferSerializers.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/BufferSerializers.java
@@ -24,6 +24,7 @@ import java.nio.ByteOrder;
import org.apache.fory.context.CopyContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.ByteBufferUtil;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.resolver.TypeResolver;
@@ -60,8 +61,14 @@ public class BufferSerializers {
MemoryBuffer newBuffer = readContext.readBufferObject();
int readerIndex = newBuffer.readerIndex();
int size = newBuffer.remaining();
+ if (size < 1) {
+ throw new DeserializationException("Invalid ByteBuffer payload");
+ }
ByteBuffer originalBuffer = newBuffer.sliceAsByteBuffer(readerIndex, size - 1);
byte isBigEndian = newBuffer.getByte(readerIndex + size - 1);
+ if (isBigEndian != 0 && isBigEndian != (byte) 1) {
+ throw new DeserializationException("Invalid ByteBuffer byte order marker");
+ }
originalBuffer.order(
isBigEndian == (byte) 1 ? ByteOrder.BIG_ENDIAN : ByteOrder.LITTLE_ENDIAN);
return originalBuffer;
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/DecimalSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/DecimalSerializer.java
index 796d67901..05043113f 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/DecimalSerializer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/DecimalSerializer.java
@@ -25,6 +25,7 @@ import java.math.MathContext;
import org.apache.fory.config.Config;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.MemoryBuffer;
/** Serializer for {@link BigDecimal} in native and xlang modes. */
@@ -32,10 +33,12 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
private static final BigInteger LONG_MIN = BigInteger.valueOf(Long.MIN_VALUE);
private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE);
private final boolean xlang;
+ private final int maxBinarySize;
public DecimalSerializer(Config config) {
super(config, BigDecimal.class);
xlang = config.isXlang();
+ maxBinarySize = config.maxBinarySize();
}
@Override
@@ -69,6 +72,8 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
int scale = buffer.readVarUInt32Small7();
int precision = buffer.readVarUInt32Small7();
int len = buffer.readVarUInt32Small7();
+ checkBinaryPayloadLength(len, maxBinarySize);
+ buffer.checkReadableBytes(len);
byte[] bytes = buffer.readBytes(len);
BigInteger bigInteger = new BigInteger(bytes);
return new BigDecimal(bigInteger, scale, new MathContext(precision));
@@ -79,7 +84,7 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
}
private BigDecimal readXlang(ReadContext readContext) {
- return readXlangDecimal(readContext.getBuffer());
+ return readXlangDecimal(readContext.getBuffer(), maxBinarySize);
}
static void writeXlangDecimal(MemoryBuffer buffer, int scale, BigInteger unscaled) {
@@ -100,13 +105,21 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
}
static BigDecimal readXlangDecimal(MemoryBuffer buffer) {
+ return readXlangDecimal(buffer, Integer.MAX_VALUE);
+ }
+
+ static BigDecimal readXlangDecimal(MemoryBuffer buffer, int maxBinarySize) {
int scale = buffer.readVarInt32();
- return new BigDecimal(readXlangUnscaled(buffer), scale);
+ return new BigDecimal(readXlangUnscaled(buffer, maxBinarySize), scale);
}
static BigInteger readXlangBigInteger(MemoryBuffer buffer) {
+ return readXlangBigInteger(buffer, Integer.MAX_VALUE);
+ }
+
+ static BigInteger readXlangBigInteger(MemoryBuffer buffer, int maxBinarySize) {
int scale = buffer.readVarInt32();
- BigInteger unscaled = readXlangUnscaled(buffer);
+ BigInteger unscaled = readXlangUnscaled(buffer, maxBinarySize);
if (scale != 0) {
throw new IllegalArgumentException(
"Cannot deserialize xlang decimal with scale " + scale + " into BigInteger");
@@ -114,7 +127,7 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
return unscaled;
}
- private static BigInteger readXlangUnscaled(MemoryBuffer buffer) {
+ private static BigInteger readXlangUnscaled(MemoryBuffer buffer, int maxBinarySize) {
long header = buffer.readVarUInt64();
if ((header & 1L) == 0L) {
return BigInteger.valueOf(decodeZigZag64(header >>> 1));
@@ -126,7 +139,12 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
throw new IllegalArgumentException(
"Invalid decimal magnitude length " + lenLong + " in xlang payload");
}
+ if (lenLong > maxBinarySize) {
+ throw new DeserializationException(
+ "Decimal magnitude length " + lenLong + " exceeds max binary size " + maxBinarySize);
+ }
int len = (int) lenLong;
+ buffer.checkReadableBytes(len);
byte[] payload = buffer.readBytes(len);
if (payload[len - 1] == 0) {
throw new IllegalArgumentException("Non-canonical decimal payload: trailing zero byte");
@@ -139,6 +157,16 @@ public final class DecimalSerializer extends ImmutableSerializer<BigDecimal> imp
return sign == 0 ? abs : abs.negate();
}
+ private static void checkBinaryPayloadLength(int len, int maxBinarySize) {
+ if (len <= 0) {
+ throw new DeserializationException("Decimal payload length must be positive: " + len);
+ }
+ if (len > maxBinarySize) {
+ throw new DeserializationException(
+ "Decimal payload length " + len + " exceeds max binary size " + maxBinarySize);
+ }
+ }
+
private static boolean canUseSmallEncoding(BigInteger value) {
if (value.compareTo(LONG_MIN) < 0 || value.compareTo(LONG_MAX) > 0) {
return false;
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java
index c714cedb6..5bbb5c822 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ExceptionSerializers.java
@@ -34,6 +34,7 @@ import org.apache.fory.config.Config;
import org.apache.fory.context.MetaReadContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.ForyException;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.memory.Platform;
import org.apache.fory.meta.TypeDef;
@@ -302,7 +303,11 @@ public final class ExceptionSerializers {
}
int indexMarker = buffer.readVarUInt32Small14();
boolean isRef = (indexMarker & 1) == 1;
+ int index = indexMarker >>> 1;
if (isRef) {
+ if (index >= metaReadContext.readTypeInfos.size) {
+ throw new ForyException("Invalid layer metadata reference id " + index);
+ }
return;
}
long typeDefId = buffer.readInt64();
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
index 3f0f60c3e..b4c0780ef 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/ObjectStreamSerializer.java
@@ -51,6 +51,7 @@ import org.apache.fory.config.Int64Encoding;
import org.apache.fory.context.MetaReadContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.ForyException;
import org.apache.fory.logging.Logger;
import org.apache.fory.logging.LoggerFactory;
import org.apache.fory.memory.MemoryBuffer;
@@ -89,9 +90,10 @@ import org.apache.fory.util.unsafe._JDKAccess;
@SuppressWarnings({"unchecked", "rawtypes"})
public class ObjectStreamSerializer extends AbstractObjectSerializer {
private static final Logger LOG = LoggerFactory.getLogger(ObjectStreamSerializer.class);
+ private static final int MAX_CACHED_TYPE_DEFS = 8192;
private final SlotInfo[] slotsInfos;
- // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots)
+ // Instance-level cache: TypeDef ID -> TypeInfo (shared across all slots).
private final LongMap<TypeInfo> typeDefIdToTypeInfo = new LongMap<>(4, 0.4f);
private static MetaSharedLayerSerializerBase<?> newGeneratedSerializer(
@@ -428,21 +430,11 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer {
TypeInfo typeInfo;
if (isRef) {
// Reference to previously read TypeInfo
- typeInfo = metaReadContext.readTypeInfos.get(index);
+ typeInfo = getMetaReadTypeInfo(metaReadContext, index);
} else {
- // New TypeDef in stream - read ID first to check cache
+ // New TypeDef in stream, with optimized reuse by validated TypeDef header.
long typeDefId = buffer.readInt64();
- typeInfo = typeDefIdToTypeInfo.get(typeDefId);
- if (typeInfo != null) {
- // Already cached - skip the TypeDef bytes, reuse existing TypeInfo
- TypeDef.skipTypeDef(buffer, typeDefId);
- } else {
- // Not cached - read full TypeDef and create TypeInfo
- TypeDef typeDef =
- typeResolver.cacheTypeDef(TypeDef.readTypeDef(typeResolver, buffer, typeDefId));
- typeInfo = new TypeInfo(senderClass, typeDef);
- typeDefIdToTypeInfo.put(typeDefId, typeInfo);
- }
+ typeInfo = readLayerTypeInfo(typeResolver, buffer, senderClass, typeDefId);
metaReadContext.readTypeInfos.add(typeInfo);
}
@@ -460,6 +452,33 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer {
skipSerializer.skipFields(readContext);
}
+ private static TypeInfo getMetaReadTypeInfo(MetaReadContext metaReadContext, int index) {
+ if (index < 0 || index >= metaReadContext.readTypeInfos.size) {
+ throw new ForyException("Invalid layer metadata reference id " + index);
+ }
+ TypeInfo typeInfo = metaReadContext.readTypeInfos.get(index);
+ if (typeInfo == null) {
+ throw new ForyException("Invalid layer metadata reference id " + index);
+ }
+ return typeInfo;
+ }
+
+ private TypeInfo readLayerTypeInfo(
+ TypeResolver typeResolver, MemoryBuffer buffer, Class<?> cls, long typeDefId) {
+ TypeInfo typeInfo = typeDefIdToTypeInfo.get(typeDefId);
+ if (typeInfo != null) {
+ TypeDef.skipTypeDef(buffer, typeDefId);
+ return typeInfo;
+ }
+ TypeDef typeDef =
+ typeResolver.cacheTypeDef(TypeDef.readTypeDef(typeResolver, buffer, typeDefId));
+ typeInfo = new TypeInfo(cls, typeDef);
+ if (typeDefIdToTypeInfo.size < MAX_CACHED_TYPE_DEFS) {
+ typeDefIdToTypeInfo.put(typeDefId, typeInfo);
+ }
+ return typeInfo;
+ }
+
private static void throwUnsupportedEncodingException(Class<?> cls)
throws UnsupportedEncodingException {
throw new UnsupportedEncodingException(
@@ -666,7 +685,7 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer {
buildFieldInfoFromObjectStreamClass(typeResolver, objectStreamClass, type);
layerTypeDef =
NativeTypeDefEncoder.buildTypeDefWithFieldInfos(
- (ClassResolver) typeResolver, type, fieldInfos, true);
+ (ClassResolver) typeResolver, type, fieldInfos);
} else {
// Fallback when ObjectStreamClass is not available (e.g., GraalVM native image)
layerTypeDef = typeResolver.getTypeDef(type, false);
@@ -842,21 +861,12 @@ public class ObjectStreamSerializer extends AbstractObjectSerializer {
int index = indexMarker >>> 1;
if (isRef) {
// Reference to previously read TypeInfo
- return metaReadContext.readTypeInfos.get(index);
+ return getMetaReadTypeInfo(metaReadContext, index);
} else {
- // New TypeDef in stream - read ID first to check cache
+ // New TypeDef in stream, with optimized reuse by validated TypeDef header.
long typeDefId = buffer.readInt64();
- TypeInfo typeInfo = typeDefIdToTypeInfo.get(typeDefId);
- if (typeInfo != null) {
- // Already cached - skip the TypeDef bytes, reuse existing TypeInfo
- TypeDef.skipTypeDef(buffer, typeDefId);
- } else {
- // Not cached - read full TypeDef and create TypeInfo
- TypeDef typeDef =
- typeResolver.cacheTypeDef(TypeDef.readTypeDef(typeResolver, buffer, typeDefId));
- typeInfo = new TypeInfo(cls, typeDef);
- typeDefIdToTypeInfo.put(typeDefId, typeInfo);
- }
+ TypeInfo typeInfo =
+ ObjectStreamSerializer.this.readLayerTypeInfo(typeResolver, buffer, cls, typeDefId);
metaReadContext.readTypeInfos.add(typeInfo);
return typeInfo;
}
diff --git a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveArraySerializers.java b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveArraySerializers.java
index 8b5a32b48..54ef7809c 100644
--- a/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveArraySerializers.java
+++ b/java/fory-core/src/main/java/org/apache/fory/serializer/PrimitiveArraySerializers.java
@@ -25,6 +25,7 @@ import org.apache.fory.config.Int64Encoding;
import org.apache.fory.context.CopyContext;
import org.apache.fory.context.ReadContext;
import org.apache.fory.context.WriteContext;
+import org.apache.fory.exception.DeserializationException;
import org.apache.fory.memory.MemoryBuffer;
import org.apache.fory.memory.Platform;
import org.apache.fory.resolver.TypeResolver;
@@ -74,13 +75,49 @@ public final class PrimitiveArraySerializers {
public abstract static class PrimitiveArraySerializer<T> extends Serializer<T>
implements Shareable {
protected final Config config;
+ protected final int maxBinarySize;
public PrimitiveArraySerializer(TypeResolver typeResolver, Class<T> cls) {
super(typeResolver.getConfig(), cls);
this.config = typeResolver.getConfig();
+ maxBinarySize = config.maxBinarySize();
}
}
+ private static void throwBinarySizeLimitExceeded(long size, int maxBinarySize) {
+ throw new DeserializationException(
+ "Binary payload size " + size + " exceeds max binary size " + maxBinarySize);
+ }
+
+ private static void throwNegativeBinarySize(int size) {
+ throw new DeserializationException("Binary payload size must be non-negative: " + size);
+ }
+
+ private static void throwNegativeElementCount(int numElements) {
+ throw new DeserializationException("Element count must be non-negative: " + numElements);
+ }
+
+ private static void throwInvalidBinarySize(int size, int maxBinarySize) {
+ if (size < 0) {
+ throwNegativeBinarySize(size);
+ } else {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
+ }
+
+ private static void throwInvalidElementCount(int numElements, int maxBinarySize, int elemSize) {
+ if (numElements < 0) {
+ throwNegativeElementCount(numElements);
+ } else {
+ throwBinarySizeLimitExceeded((long) numElements * elemSize, maxBinarySize);
+ }
+ }
+
+ private static void throwUnalignedBinarySize(int size, int elemSize) {
+ throw new DeserializationException(
+ "Binary payload size " + size + " is not aligned to element size " + elemSize);
+ }
+
public static final class BooleanArraySerializer extends PrimitiveArraySerializer<boolean[]> {
public BooleanArraySerializer(TypeResolver typeResolver) {
super(typeResolver, boolean[].class);
@@ -109,11 +146,20 @@ public final class PrimitiveArraySerializers {
if (readContext.isPeerOutOfBandEnabled()) {
MemoryBuffer buf = readContext.readBufferObject();
int size = buf.remaining();
+ if (size > maxBinarySize) {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
boolean[] values = new boolean[size];
buf.copyToUnsafe(0, values, Platform.BOOLEAN_ARRAY_OFFSET, size);
return values;
}
int size = buffer.readVarUInt32Small7();
+ if (size < 0 || size > maxBinarySize) {
+ throwInvalidBinarySize(size, maxBinarySize);
+ }
+ if (size > buffer.remaining()) {
+ buffer.checkReadableBytes(size);
+ }
boolean[] values = new boolean[size];
buffer.readToUnsafe(values, Platform.BOOLEAN_ARRAY_OFFSET, size);
return values;
@@ -148,11 +194,20 @@ public final class PrimitiveArraySerializers {
if (readContext.isPeerOutOfBandEnabled()) {
MemoryBuffer buf = readContext.readBufferObject();
int size = buf.remaining();
+ if (size > maxBinarySize) {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
byte[] values = new byte[size];
buf.copyToUnsafe(0, values, Platform.BYTE_ARRAY_OFFSET, size);
return values;
}
int size = buffer.readVarUInt32Small7();
+ if (size < 0 || size > maxBinarySize) {
+ throwInvalidBinarySize(size, maxBinarySize);
+ }
+ if (size > buffer.remaining()) {
+ buffer.checkReadableBytes(size);
+ }
byte[] values = new byte[size];
buffer.readToUnsafe(values, Platform.BYTE_ARRAY_OFFSET, size);
return values;
@@ -208,7 +263,13 @@ public final class PrimitiveArraySerializers {
if (readContext.isPeerOutOfBandEnabled()) {
MemoryBuffer buf = readContext.readBufferObject();
int size = buf.remaining();
- int numElements = size / 2;
+ if ((size & 1) != 0) {
+ throwUnalignedBinarySize(size, 2);
+ }
+ if (size > maxBinarySize) {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
+ int numElements = size >>> 1;
char[] values = new char[numElements];
if (Platform.IS_LITTLE_ENDIAN) {
buf.copyToUnsafe(0, values, Platform.CHAR_ARRAY_OFFSET, size);
@@ -218,7 +279,16 @@ public final class PrimitiveArraySerializers {
return values;
}
int size = buffer.readVarUInt32Small7();
- int numElements = size / 2;
+ if ((size & 1) != 0) {
+ throwUnalignedBinarySize(size, 2);
+ }
+ if (size < 0 || size > maxBinarySize) {
+ throwInvalidBinarySize(size, maxBinarySize);
+ }
+ if (size > buffer.remaining()) {
+ buffer.checkReadableBytes(size);
+ }
+ int numElements = size >>> 1;
char[] values = new char[numElements];
if (Platform.IS_LITTLE_ENDIAN) {
buffer.readToUnsafe(values, Platform.CHAR_ARRAY_OFFSET, size);
@@ -256,7 +326,7 @@ public final class PrimitiveArraySerializers {
@Override
public short[] read(ReadContext readContext) {
- return readShortBits(readContext);
+ return readShortBits(readContext, maxBinarySize);
}
}
@@ -307,7 +377,13 @@ public final class PrimitiveArraySerializers {
if (readContext.isPeerOutOfBandEnabled()) {
MemoryBuffer buf = readContext.readBufferObject();
int size = buf.remaining();
- int numElements = size / 4;
+ if ((size & 3) != 0) {
+ throwUnalignedBinarySize(size, 4);
+ }
+ if (size > maxBinarySize) {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
+ int numElements = size >>> 2;
int[] values = new int[numElements];
if (size > 0) {
if (Platform.IS_LITTLE_ENDIAN) {
@@ -322,7 +398,16 @@ public final class PrimitiveArraySerializers {
return readInt32Compressed(buffer);
}
int size = buffer.readVarUInt32Small7();
- int numElements = size / 4;
+ if ((size & 3) != 0) {
+ throwUnalignedBinarySize(size, 4);
+ }
+ if (size < 0 || size > maxBinarySize) {
+ throwInvalidBinarySize(size, maxBinarySize);
+ }
+ if (size > buffer.remaining()) {
+ buffer.checkReadableBytes(size);
+ }
+ int numElements = size >>> 2;
int[] values = new int[numElements];
if (size > 0) {
if (Platform.IS_LITTLE_ENDIAN) {
@@ -353,6 +438,9 @@ public final class PrimitiveArraySerializers {
private int[] readInt32Compressed(MemoryBuffer buffer) {
int numElements = buffer.readVarUInt32Small7();
+ if (numElements < 0 || numElements > maxBinarySize / 4) {
+ throwInvalidElementCount(numElements, maxBinarySize, 4);
+ }
int[] values = new int[numElements];
for (int i = 0; i < numElements; i++) {
values[i] = buffer.readVarInt32();
@@ -414,7 +502,13 @@ public final class PrimitiveArraySerializers {
if (readContext.isPeerOutOfBandEnabled()) {
MemoryBuffer buf = readContext.readBufferObject();
int size = buf.remaining();
- int numElements = size / 8;
+ if ((size & 7) != 0) {
+ throwUnalignedBinarySize(size, 8);
+ }
+ if (size > maxBinarySize) {
+ throwBinarySizeLimitExceeded(size, maxBinarySize);
+ }
+ int numElements = size >>> 3;
long[] values = new long[numElements];
if (size > 0) {
if (Platform.IS_LITTLE_ENDIAN) {
@@ -429,7 +523,16 @@ public final class PrimitiveArraySerializers {
return readInt64Compressed(buffer, config.longEncoding());
}
int size = buffer.readVarUInt32Small7();
- int numElements = size / 8;
+ if ((size & 7) != 0) {
+ throwUnalignedBinarySize(size, 8);
+ }
... 13638 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org