You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by ch...@apache.org on 2025/09/24 05:00:39 UTC
(fory) branch main updated: feat(rust): support Map and register_by_name (#2649)
This is an automated email from the ASF dual-hosted git repository.
chaokunyang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fory.git
The following commit(s) were added to refs/heads/main by this push:
new bd7b369d6 feat(rust): support Map and register_by_name (#2649)
bd7b369d6 is described below
commit bd7b369d60f5f321406cc773f580fe9efa343b81
Author: urlyy <li...@gmail.com>
AuthorDate: Wed Sep 24 13:00:36 2025 +0800
feat(rust): support Map and register_by_name (#2649)
## What does this PR do?
1. Refactored the code to reduce function reuse, making it easier to
maintain and extend later.
2. Added support for handling `Map`.
3. Added support for `register_by_name`.
4. Add new alignment tests with Java.
5. Support handle string when not string_compressed.
6. Support int <-> Integer conversion.
7. Add skip meta bytes after deserialize.
## Does this PR introduce any user-facing change?
- [ ] Does this PR introduce any public API change?
- [ ] Does this PR introduce any binary protocol compatibility change?
---------
Co-authored-by: Shawn Yang <ch...@apache.org>
Co-authored-by: Shawn Yang <sh...@gmail.com>
---
.../test/java/org/apache/fory/RustXlangTest.java | 254 ++++++++++++++++---
rust/fory-core/src/buffer.rs | 4 +
rust/fory-core/src/fory.rs | 57 +++--
rust/fory-core/src/meta/meta_string.rs | 174 ++++++++++---
rust/fory-core/src/meta/mod.rs | 7 +-
rust/fory-core/src/meta/type_meta.rs | 104 +++++---
rust/fory-core/src/resolver/context.rs | 4 +-
rust/fory-core/src/resolver/meta_resolver.rs | 3 +-
rust/fory-core/src/resolver/type_resolver.rs | 115 +++++----
rust/fory-core/src/serializer/bool.rs | 14 +-
rust/fory-core/src/serializer/collection.rs | 111 ++++----
rust/fory-core/src/serializer/datetime.rs | 42 ++--
rust/fory-core/src/serializer/list.rs | 18 +-
rust/fory-core/src/serializer/map.rs | 214 ++++++++++++----
rust/fory-core/src/serializer/mod.rs | 70 ++++--
rust/fory-core/src/serializer/number.rs | 14 +-
rust/fory-core/src/serializer/option.rs | 21 +-
rust/fory-core/src/serializer/primitive_list.rs | 25 +-
rust/fory-core/src/serializer/set.rs | 18 +-
rust/fory-core/src/serializer/skip.rs | 115 ++++++---
rust/fory-core/src/serializer/string.rs | 37 ++-
rust/fory-core/src/types.rs | 12 +-
rust/fory-derive/src/object/derive_enum.rs | 74 ++++--
rust/fory-derive/src/object/misc.rs | 23 +-
rust/fory-derive/src/object/read.rs | 52 ++--
rust/fory-derive/src/object/serializer.rs | 4 +-
rust/fory-derive/src/object/util.rs | 214 +++++++++++++---
rust/fory-derive/src/object/write.rs | 29 ++-
rust/tests/tests/test_compatible.rs | 195 ++++++++++++--
rust/tests/tests/test_cross_language.rs | 279 ++++++++++++++++++---
rust/tests/tests/test_meta_string.rs | 85 ++++---
31 files changed, 1810 insertions(+), 578 deletions(-)
diff --git a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
index 6e02e4c82..78397794a 100644
--- a/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
+++ b/java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java
@@ -81,7 +81,6 @@ public class RustXlangTest extends ForyTestBase {
@BeforeClass
public void isRustJavaCIEnabled() {
String enabled = System.getenv("FORY_RUST_JAVA_CI");
- enabled = "0"; // skip rust ci
if (enabled == null || !enabled.equals("1")) {
throw new SkipException("Skipping RustXlangTest: FORY_RUST_JAVA_CI not set to 1");
}
@@ -115,6 +114,14 @@ public class RustXlangTest extends ForyTestBase {
testCrossLanguageSerializer(Language.RUST, command);
command.set(RUST_TESTCASE_INDEX, "test_simple_struct");
testSimpleStruct(Language.RUST, command);
+ command.set(RUST_TESTCASE_INDEX, "test_simple_named_struct");
+ testSimpleNamedStruct(Language.RUST, command);
+ command.set(RUST_TESTCASE_INDEX, "test_list");
+ testList(Language.RUST, command);
+ command.set(RUST_TESTCASE_INDEX, "test_map");
+ testMap(Language.RUST, command);
+ command.set(RUST_TESTCASE_INDEX, "test_integer");
+ testInteger(Language.RUST, command);
}
@Test
@@ -291,15 +298,9 @@ public class RustXlangTest extends ForyTestBase {
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
}
- private void testStringSerializer(Language language, List<String> command) throws Exception {
- Path dataFile = Files.createTempFile("test_string_serializer", "data");
+ private void _testStringSerializer(
+ Fory fory, Path dataFile, Language language, List<String> command) throws IOException {
MemoryBuffer buffer = MemoryUtils.buffer(100);
- Fory fory =
- Fory.builder()
- .withStringCompressed(true)
- .withWriteNumUtf16BytesForUtf8Encoding(false)
- .requireClassRegistration(false)
- .build();
StringSerializer serializer = new StringSerializer(fory);
String[] testStrings =
new String[] {
@@ -320,15 +321,31 @@ public class RustXlangTest extends ForyTestBase {
Pair<Map<String, String>, File> env_workdir =
setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
-
buffer = MemoryUtils.wrap(Files.readAllBytes(dataFile));
-
for (String expected : testStrings) {
String actual = serializer.readJavaString(buffer);
Assert.assertEquals(actual, expected);
}
}
+ private void testStringSerializer(Language language, List<String> command) throws Exception {
+ Path dataFile = Files.createTempFile("test_string_serializer", "data");
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+ _testStringSerializer(fory, dataFile, language, command);
+ Fory foryCompress =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withStringCompressed(true)
+ .withWriteNumUtf16BytesForUtf8Encoding(false)
+ .build();
+ _testStringSerializer(foryCompress, dataFile, language, command);
+ }
+
enum Color {
Green,
Red,
@@ -348,10 +365,7 @@ public class RustXlangTest extends ForyTestBase {
Fory fory =
Fory.builder()
.withLanguage(Language.XLANG)
- .requireClassRegistration(false)
- .withStringCompressed(true)
.withCompatibleMode(CompatibleMode.COMPATIBLE)
- .withWriteNumUtf16BytesForUtf8Encoding(false)
.build();
fory.register(Color.class, 101);
MemoryBuffer buffer = MemoryUtils.buffer(32);
@@ -373,7 +387,6 @@ public class RustXlangTest extends ForyTestBase {
fory.serialize(buffer, day);
Instant instant = Instant.ofEpochSecond(100);
fory.serialize(buffer, instant);
- // test primitive arrays
fory.serialize(buffer, new boolean[] {true, false});
fory.serialize(buffer, new short[] {1, Short.MAX_VALUE});
fory.serialize(buffer, new int[] {1, Integer.MAX_VALUE});
@@ -390,8 +403,7 @@ public class RustXlangTest extends ForyTestBase {
// map.put(list.get(i), list.get(i));
// }
// fory.serialize(buffer, map);
- // System.out.println("bytes: " + Arrays.toString(buffer.getBytes(0,
- // buffer.writerIndex())));
+
// Set<Object> set = new HashSet<>(list);
// fory.serialize(buffer, set);
@@ -443,11 +455,15 @@ public class RustXlangTest extends ForyTestBase {
@Data
static class SimpleStruct {
- // int f2;
+ HashMap<Integer, Double> f1;
+ int f2;
Item f3;
String f4;
- Color f5;
- // int last;
+ // Color f5;
+ List<String> f6;
+ int f7;
+ // Integer f8;
+ // Integer last;
}
private void testSimpleStruct(Language language, List<String> command)
@@ -455,31 +471,207 @@ public class RustXlangTest extends ForyTestBase {
Fory fory =
Fory.builder()
.withLanguage(Language.XLANG)
- .withRefTracking(false)
.withCompatibleMode(CompatibleMode.COMPATIBLE)
.withCodegen(false)
- .withStringCompressed(true)
- .withWriteNumUtf16BytesForUtf8Encoding(false)
.build();
fory.register(Color.class, 101);
fory.register(Item.class, 102);
fory.register(SimpleStruct.class, 103);
+
Item item = new Item();
item.name = "item";
+ HashMap<Integer, Double> f1 = new HashMap<>();
+ f1.put(1, 1.0);
+ f1.put(2, 2.0);
SimpleStruct obj = new SimpleStruct();
- // obj.f2 = 10;
+ obj.f1 = f1;
+ obj.f2 = 39;
obj.f3 = item;
- obj.f4 = "f3";
- obj.f5 = Color.White;
+ obj.f4 = "f4";
+ // obj.f5 = Color.White;
+ obj.f6 = Collections.singletonList("f6");
+ obj.f7 = 40;
+ // obj.f8 = 41;
// obj.last = 42;
- byte[] serialized = fory.serialize(obj);
- Assert.assertEquals(fory.deserialize(serialized), obj);
- System.out.println(Arrays.toString(serialized));
+
+ MemoryBuffer buffer = MemoryUtils.buffer(32);
+ fory.serialize(buffer, obj);
+
Path dataFile = Files.createTempFile("test_simple_struct", "data");
Pair<Map<String, String>, File> env_workdir =
- setFilePath(language, command, dataFile, serialized);
+ setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
+ Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ Assert.assertEquals(fory.deserialize(buffer2), obj);
+ }
+
+ private void testSimpleNamedStruct(Language language, List<String> command)
+ throws java.io.IOException {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withCodegen(false)
+ .build();
+ fory.register(Color.class, "demo", "color");
+ fory.register(Item.class, "demo", "item");
+ fory.register(SimpleStruct.class, "demo", "simple_struct");
+
+ Item item = new Item();
+ item.name = "item";
+ HashMap<Integer, Double> f1 = new HashMap<>();
+ f1.put(1, 1.0);
+ f1.put(2, 2.0);
+ SimpleStruct obj = new SimpleStruct();
+ obj.f1 = f1;
+ obj.f2 = 39;
+ obj.f3 = item;
+ obj.f4 = "f4";
+ // obj.f5 = Color.White;
+ obj.f6 = Collections.singletonList("f6");
+ obj.f7 = 40;
+ // obj.f8 = 41;
+ // obj.last = 42;
+
+ MemoryBuffer buffer = MemoryUtils.buffer(32);
+ fory.serialize(buffer, obj);
+
+ Path dataFile = Files.createTempFile("test_named_simple_struct", "data");
+ Pair<Map<String, String>, File> env_workdir =
+ setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
+ Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ Assert.assertEquals(fory.deserialize(buffer2), obj);
+ }
+
+ private void testList(Language language, List<String> command) throws java.io.IOException {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withCodegen(false)
+ .build();
+ fory.register(Item.class, 102);
+ MemoryBuffer buffer = MemoryUtils.buffer(32);
+ List<String> strList = Arrays.asList("a", "b");
+ List<String> strList2 = Arrays.asList(null, "b");
+ Item item = new Item();
+ item.name = "a";
+ Item item2 = new Item();
+ item2.name = "b";
+ Item item3 = new Item();
+ item3.name = "c";
+ List<Item> itemList = Arrays.asList(item, item2);
+ List<Item> itemList2 = Arrays.asList(null, item3);
+ fory.serialize(buffer, strList);
+ fory.serialize(buffer, strList2);
+ fory.serialize(buffer, itemList);
+ fory.serialize(buffer, itemList2);
+
+ Path dataFile = Files.createTempFile("test_list", "data");
+ Pair<Map<String, String>, File> env_workdir =
+ setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
+ Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ Assert.assertEquals(fory.deserialize(buffer2), strList);
+ Assert.assertEquals(fory.deserialize(buffer2), strList2);
+ Assert.assertEquals(fory.deserialize(buffer2), itemList);
+ Assert.assertEquals(fory.deserialize(buffer2), itemList2);
+ }
+
+ private void testMap(Language language, List<String> command) throws java.io.IOException {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .withCodegen(false)
+ .build();
+ fory.register(Item.class, 102);
+ MemoryBuffer buffer = MemoryUtils.buffer(32);
+ Map<String, String> strMap = new HashMap<>();
+ strMap.put("k1", "v1");
+ strMap.put(null, "v2");
+ strMap.put("k3", null);
+ strMap.put("k4", "v4");
+ Item item = new Item();
+ item.name = "item1";
+ Item item2 = new Item();
+ item2.name = "item2";
+ Item item3 = new Item();
+ item3.name = "item3";
+ Map<String, Item> itemMap = new HashMap<>();
+ itemMap.put("k1", item);
+ itemMap.put(null, item2);
+ itemMap.put("k3", null);
+ itemMap.put("k4", item3);
+ fory.serialize(buffer, strMap);
+ fory.serialize(buffer, itemMap);
+ Path dataFile = Files.createTempFile("test_map", "data");
+ Pair<Map<String, String>, File> env_workdir =
+ setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
+ Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ Assert.assertEquals(fory.deserialize(buffer2), strMap);
+ Assert.assertEquals(fory.deserialize(buffer2), itemMap);
+ }
+
+ static class Item1 {
+ int f1;
+ int f2;
+ Integer f3;
+ Integer f4;
+ Integer f5;
+ Integer f6;
+ }
+
+ private void testInteger(Language language, List<String> command) throws java.io.IOException {
+ Fory fory =
+ Fory.builder()
+ .withLanguage(Language.XLANG)
+ .withCompatibleMode(CompatibleMode.COMPATIBLE)
+ .build();
+ fory.register(Item1.class, 101);
+ Item1 item1 = new Item1();
+ int f1 = 1;
+ int f2 = 2;
+ Integer f3 = 3;
+ Integer f4 = 4;
+ Integer f5 = null;
+ Integer f6 = null;
+ item1.f1 = f1;
+ item1.f2 = f2;
+ item1.f3 = f3;
+ item1.f4 = f4;
+ item1.f5 = f5;
+ item1.f6 = f6;
+ MemoryBuffer buffer = MemoryBuffer.newHeapBuffer(32);
+ fory.serialize(buffer, item1);
+ fory.serialize(buffer, f1);
+ fory.serialize(buffer, f2);
+ fory.serialize(buffer, f3);
+ fory.serialize(buffer, f4);
+ fory.serialize(buffer, f5);
+ fory.serialize(buffer, f6);
+
+ Path dataFile = Files.createTempFile("test_integer", "data");
+ Pair<Map<String, String>, File> env_workdir =
+ setFilePath(language, command, dataFile, buffer.getBytes(0, buffer.writerIndex()));
Assert.assertTrue(executeCommand(command, 30, env_workdir.getLeft(), env_workdir.getRight()));
- Assert.assertEquals(fory.deserialize(Files.readAllBytes(dataFile)), obj);
+
+ MemoryBuffer buffer2 = MemoryUtils.wrap(Files.readAllBytes(dataFile));
+ // Item1 newItem1 = (Item1) fory.deserialize(buffer2);
+ // Assert.assertEquals(newItem1.f1, 1);
+ // Assert.assertEquals(newItem1.f2, 2);
+ // Assert.assertEquals(newItem1.f3, 3);
+ // Assert.assertEquals(newItem1.f4, 4);
+ // Assert.assertEquals(newItem1.f5, 0);
+ // Assert.assertNull(newItem1.f6);
+ Assert.assertEquals(fory.deserialize(buffer2), 1);
+ Assert.assertEquals(fory.deserialize(buffer2), 2);
+ Assert.assertEquals(fory.deserialize(buffer2), 3);
+ Assert.assertEquals(fory.deserialize(buffer2), 4);
+ Assert.assertEquals(fory.deserialize(buffer2), 0);
+ Assert.assertNull(fory.deserialize(buffer2));
}
/**
diff --git a/rust/fory-core/src/buffer.rs b/rust/fory-core/src/buffer.rs
index 78df85a87..87ddeb163 100644
--- a/rust/fory-core/src/buffer.rs
+++ b/rust/fory-core/src/buffer.rs
@@ -311,6 +311,10 @@ impl<'bf> Reader<'bf> {
&self.bf[self.cursor..self.bf.len()]
}
+ pub fn get_cursor(&self) -> usize {
+ self.cursor
+ }
+
pub fn u8(&mut self) -> u8 {
let result = self.bf[self.cursor];
self.move_next(1);
diff --git a/rust/fory-core/src/fory.rs b/rust/fory-core/src/fory.rs
index baa0808a3..4a7412176 100644
--- a/rust/fory-core/src/fory.rs
+++ b/rust/fory-core/src/fory.rs
@@ -18,7 +18,6 @@
use crate::buffer::{Reader, Writer};
use crate::ensure;
use crate::error::Error;
-use crate::meta::MetaStringEncoder;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
use crate::resolver::type_resolver::{TypeInfo, TypeResolver};
@@ -34,6 +33,7 @@ pub struct Fory {
mode: Mode,
xlang: bool,
type_resolver: TypeResolver,
+ compress_string: bool,
}
impl Default for Fory {
@@ -42,6 +42,7 @@ impl Default for Fory {
mode: Mode::SchemaConsistent,
xlang: true,
type_resolver: TypeResolver::default(),
+ compress_string: false,
}
}
}
@@ -57,10 +58,19 @@ impl Fory {
self
}
+ pub fn compress_string(mut self, compress_string: bool) -> Self {
+ self.compress_string = compress_string;
+ self
+ }
+
pub fn get_mode(&self) -> &Mode {
&self.mode
}
+ pub fn is_compress_string(&self) -> bool {
+ self.compress_string
+ }
+
pub fn write_head<T: Serializer>(&self, is_none: bool, writer: &mut Writer) {
const HEAD_SIZE: usize = 10;
writer.reserve(<T as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE + HEAD_SIZE);
@@ -136,13 +146,18 @@ impl Fory {
if is_none {
return Ok(T::default());
}
+ let mut bytes_to_skip = 0;
if self.mode == Mode::Compatible {
let meta_offset = context.reader.i32();
if meta_offset != -1 {
- context.load_meta(meta_offset as usize);
+ bytes_to_skip = context.load_meta(meta_offset as usize);
}
}
- <T as Serializer>::deserialize(context, false)
+ let result = <T as Serializer>::deserialize(context, false);
+ if bytes_to_skip > 0 {
+ context.reader.skip(bytes_to_skip as u32);
+ }
+ result
}
pub fn serialize<T: Serializer>(&self, record: &T) -> Vec<u8> {
@@ -176,34 +191,24 @@ impl Fory {
}
pub fn register<T: 'static + StructSerializer>(&mut self, id: u32) {
- let actual_type_id = T::actual_type_id(id);
- let empty_bytes = "".as_bytes().to_vec();
- let type_info = TypeInfo::new::<T>(
- self,
- actual_type_id,
- empty_bytes.clone(),
- empty_bytes,
- false,
- );
- self.type_resolver.register::<T>(type_info);
+ let actual_type_id = T::actual_type_id(id, false, &self.mode);
+ let empty_string = String::new();
+ let type_info =
+ TypeInfo::new::<T>(self, actual_type_id, &empty_string, &empty_string, false);
+ self.type_resolver.register::<T>(&type_info);
}
- pub fn register_by_name<T: 'static + StructSerializer>(
+ pub fn register_by_namespace<T: 'static + StructSerializer>(
&mut self,
namespace: &str,
type_name: &str,
) {
- let type_id = T::actual_type_id(0);
- let encoder = MetaStringEncoder::new();
- let namespace_metastring = encoder.encode(namespace).unwrap();
- let type_name_metastring = encoder.encode(type_name).unwrap();
- let type_info = TypeInfo::new::<T>(
- self,
- type_id,
- namespace_metastring.bytes,
- type_name_metastring.bytes,
- true,
- );
- self.type_resolver.register::<T>(type_info);
+ let actual_type_id = T::actual_type_id(0, true, &self.mode);
+ let type_info = TypeInfo::new::<T>(self, actual_type_id, namespace, type_name, true);
+ self.type_resolver.register::<T>(&type_info);
+ }
+
+ pub fn register_by_name<T: 'static + StructSerializer>(&mut self, type_name: &str) {
+ self.register_by_namespace::<T>("", type_name);
}
}
diff --git a/rust/fory-core/src/meta/meta_string.rs b/rust/fory-core/src/meta/meta_string.rs
index 05205ea81..4e8916f36 100644
--- a/rust/fory-core/src/meta/meta_string.rs
+++ b/rust/fory-core/src/meta/meta_string.rs
@@ -25,9 +25,18 @@ use crate::meta::string_util;
const SHORT_MAX_VALUE: usize = 32767;
// const HEADER_MASK:i64 = 0xff;
-#[derive(Debug, PartialEq, Hash, Eq, Clone)]
+pub static NAMESPACE_ENCODER: MetaStringEncoder = MetaStringEncoder::new('.', '_');
+pub static TYPE_NAME_ENCODER: MetaStringEncoder = MetaStringEncoder::new('$', '_');
+pub static FIELD_NAME_ENCODER: MetaStringEncoder = MetaStringEncoder::new('$', '_');
+
+pub static NAMESPACE_DECODER: MetaStringDecoder = MetaStringDecoder::new('.', '_');
+pub static FIELD_NAME_DECODER: MetaStringDecoder = MetaStringDecoder::new('$', '_');
+pub static TYPE_NAME_DECODER: MetaStringDecoder = MetaStringDecoder::new('$', '_');
+
+#[derive(Debug, PartialEq, Hash, Eq, Clone, Copy, Default)]
#[repr(i16)]
pub enum Encoding {
+ #[default]
Utf8 = 0x00,
LowerSpecial = 0x01,
LowerUpperDigitSpecial = 0x02,
@@ -59,16 +68,24 @@ impl MetaStringBytes {
}
}
-#[derive(Debug, PartialEq, Hash, Eq, Clone)]
+#[derive(Debug, PartialEq, Hash, Eq, Clone, Default)]
pub struct MetaString {
pub original: String,
pub encoding: Encoding,
pub bytes: Vec<u8>,
pub strip_last_char: bool,
+ pub special_char1: char,
+ pub special_char2: char,
}
impl MetaString {
- pub fn new(original: String, encoding: Encoding, bytes: Vec<u8>) -> Result<Self, Error> {
+ pub fn new(
+ original: String,
+ encoding: Encoding,
+ bytes: Vec<u8>,
+ special_char1: char,
+ special_char2: char,
+ ) -> Result<Self, Error> {
let mut strip_last_char = false;
if encoding != Encoding::Utf8 {
ensure!(!bytes.is_empty(), anyhow!("Encoded data cannot be empty"));
@@ -80,20 +97,22 @@ impl MetaString {
encoding,
bytes,
strip_last_char,
+ special_char1,
+ special_char2,
})
}
}
-pub struct MetaStringDecoder {}
-impl Default for MetaStringDecoder {
- fn default() -> Self {
- Self::new()
- }
+#[derive(Clone)]
+pub struct MetaStringDecoder {
+ pub special_char1: char,
+ pub special_char2: char,
}
-#[derive(Default)]
-pub struct MetaStringEncoder<'a> {
- encoding_options: Option<&'a [Encoding]>,
+#[derive(Clone)]
+pub struct MetaStringEncoder {
+ pub special_char1: char,
+ pub special_char2: char,
}
#[derive(Debug)]
@@ -104,23 +123,29 @@ struct StringStatistics {
can_lower_special_encoded: bool,
}
-impl<'a> MetaStringEncoder<'a> {
- pub fn new() -> Self {
- Self::default()
- }
- pub fn set_options(mut self, encoding_options: Option<&'a [Encoding]>) -> Self {
- self.encoding_options = encoding_options;
- self
+impl MetaStringEncoder {
+ pub const fn new(special_char1: char, special_char2: char) -> Self {
+ Self {
+ special_char1,
+ special_char2,
+ }
}
fn is_latin(&self, s: &str) -> bool {
string_util::is_latin(s)
}
- pub fn encode(&self, input: &str) -> Result<MetaString, Error> {
+ fn _encode(&self, input: &str) -> Result<Option<MetaString>, Error> {
if input.is_empty() {
- return MetaString::new(input.to_string(), Encoding::Utf8, vec![]);
+ return Ok(Some(MetaString::new(
+ input.to_string(),
+ Encoding::Utf8,
+ vec![],
+ self.special_char1,
+ self.special_char2,
+ )?));
}
+
ensure!(
input.len() < SHORT_MAX_VALUE,
anyhow!(
@@ -128,15 +153,42 @@ impl<'a> MetaStringEncoder<'a> {
input.len()
)
);
+
if !self.is_latin(input) {
- return MetaString::new(input.to_string(), Encoding::Utf8, input.as_bytes().to_vec());
+ return Ok(Some(MetaString::new(
+ input.to_string(),
+ Encoding::Utf8,
+ input.as_bytes().to_vec(),
+ self.special_char1,
+ self.special_char2,
+ )?));
+ }
+
+ Ok(None)
+ }
+
+ pub fn encode(&self, input: &str) -> Result<MetaString, Error> {
+ if let Some(ms) = self._encode(input)? {
+ return Ok(ms);
}
- let encoding = self.compute_encoding(input);
+ let encoding = self.compute_encoding(input, None);
self.encode_with_encoding(input, encoding)
}
- fn compute_encoding(&self, input: &str) -> Encoding {
- let allow = |e: Encoding| self.encoding_options.map_or(true, |opts| opts.contains(&e));
+ pub fn encode_with_encodings(
+ &self,
+ input: &str,
+ encodings: &[Encoding],
+ ) -> Result<MetaString, Error> {
+ if let Some(ms) = self._encode(input)? {
+ return Ok(ms);
+ }
+ let encoding = self.compute_encoding(input, Some(encodings));
+ self.encode_with_encoding(input, encoding)
+ }
+
+ fn compute_encoding(&self, input: &str, encodings: Option<&[Encoding]>) -> Encoding {
+ let allow = |e: Encoding| encodings.map_or(true, |opts| opts.contains(&e));
let statistics = self.compute_statistics(input);
if statistics.can_lower_special_encoded && allow(Encoding::LowerSpecial) {
return Encoding::LowerSpecial;
@@ -174,8 +226,7 @@ impl<'a> MetaStringEncoder<'a> {
&& !(c.is_lowercase()
|| c.is_uppercase()
|| c.is_ascii_digit()
- || c == '.'
- || c == '_')
+ || (c == self.special_char1 || c == self.special_char2))
{
can_lower_upper_digit_special_encoded = false;
}
@@ -205,7 +256,13 @@ impl<'a> MetaStringEncoder<'a> {
encoding: Encoding,
) -> Result<MetaString, Error> {
if input.is_empty() {
- return MetaString::new(input.to_string(), Encoding::Utf8, vec![]);
+ return MetaString::new(
+ input.to_string(),
+ Encoding::Utf8,
+ vec![],
+ self.special_char1,
+ self.special_char2,
+ );
}
ensure!(
input.len() < SHORT_MAX_VALUE,
@@ -220,30 +277,66 @@ impl<'a> MetaStringEncoder<'a> {
);
if input.is_empty() {
- return MetaString::new(input.to_string(), Encoding::Utf8, vec![]);
+ return MetaString::new(
+ input.to_string(),
+ Encoding::Utf8,
+ vec![],
+ self.special_char1,
+ self.special_char2,
+ );
};
match encoding {
Encoding::LowerSpecial => {
let encoded_data = self.encode_lower_special(input)?;
- MetaString::new(input.to_string(), encoding, encoded_data)
+ MetaString::new(
+ input.to_string(),
+ encoding,
+ encoded_data,
+ self.special_char1,
+ self.special_char2,
+ )
}
Encoding::LowerUpperDigitSpecial => {
let encoded_data = self.encode_lower_upper_digit_special(input)?;
- MetaString::new(input.to_string(), encoding, encoded_data)
+ MetaString::new(
+ input.to_string(),
+ encoding,
+ encoded_data,
+ self.special_char1,
+ self.special_char2,
+ )
}
Encoding::FirstToLowerSpecial => {
let encoded_data = self.encode_first_to_lower_special(input)?;
- MetaString::new(input.to_string(), encoding, encoded_data)
+ MetaString::new(
+ input.to_string(),
+ encoding,
+ encoded_data,
+ self.special_char1,
+ self.special_char2,
+ )
}
Encoding::AllToLowerSpecial => {
let upper_count = input.chars().filter(|c| c.is_uppercase()).count();
let encoded_data = self.encode_all_to_lower_special(input, upper_count)?;
- MetaString::new(input.to_string(), encoding, encoded_data)
+ MetaString::new(
+ input.to_string(),
+ encoding,
+ encoded_data,
+ self.special_char1,
+ self.special_char2,
+ )
}
Encoding::Utf8 => {
let encoded_data = input.as_bytes().to_vec();
- MetaString::new(input.to_string(), Encoding::Utf8, encoded_data)
+ MetaString::new(
+ input.to_string(),
+ Encoding::Utf8,
+ encoded_data,
+ self.special_char1,
+ self.special_char2,
+ )
}
}
}
@@ -317,9 +410,9 @@ impl<'a> MetaStringEncoder<'a> {
'A'..='Z' => Ok(c as u8 - b'A' + 26),
'0'..='9' => Ok(c as u8 - b'0' + 52),
_ => {
- if c == '.' {
+ if c == self.special_char1 {
Ok(62)
- } else if c == '_' {
+ } else if c == self.special_char2 {
Ok(63)
} else {
Err(anyhow!(
@@ -334,8 +427,11 @@ impl<'a> MetaStringEncoder<'a> {
}
impl MetaStringDecoder {
- pub fn new() -> Self {
- MetaStringDecoder {}
+ pub const fn new(special_char1: char, special_char2: char) -> Self {
+ MetaStringDecoder {
+ special_char1,
+ special_char2,
+ }
}
pub fn decode(&self, encoded_data: &[u8], encoding: Encoding) -> Result<String, Error> {
@@ -424,8 +520,8 @@ impl MetaStringDecoder {
0..=25 => Ok((b'a' + char_value) as char),
26..=51 => Ok((b'A' + char_value - 26) as char),
52..=61 => Ok((b'0' + char_value - 52) as char),
- 62 => Ok('.'),
- 63 => Ok('_'),
+ 62 => Ok(self.special_char1),
+ 63 => Ok(self.special_char2),
_ => Err(anyhow!(
"Invalid character value for LOWER_UPPER_DIGIT_SPECIAL decoding: {char_value}"
))?,
diff --git a/rust/fory-core/src/meta/mod.rs b/rust/fory-core/src/meta/mod.rs
index ffc3931f0..4eea1ae6b 100644
--- a/rust/fory-core/src/meta/mod.rs
+++ b/rust/fory-core/src/meta/mod.rs
@@ -21,6 +21,11 @@ mod type_meta;
pub use meta_string::{
Encoding, MetaString, MetaStringBytes, MetaStringDecoder, MetaStringEncoder,
+ FIELD_NAME_DECODER, FIELD_NAME_ENCODER, NAMESPACE_DECODER, NAMESPACE_ENCODER,
+ TYPE_NAME_DECODER, TYPE_NAME_ENCODER,
};
pub use string_util::{get_latin1_length, is_latin, murmurhash3_x64_128};
-pub use type_meta::{FieldInfo, FieldType, NullableFieldType, TypeMeta, TypeMetaLayer};
+pub use type_meta::{
+ FieldInfo, FieldType, NullableFieldType, TypeMeta, TypeMetaLayer, NAMESPACE_ENCODINGS,
+ TYPE_NAME_ENCODINGS,
+};
diff --git a/rust/fory-core/src/meta/type_meta.rs b/rust/fory-core/src/meta/type_meta.rs
index eef783f20..13c3b4042 100644
--- a/rust/fory-core/src/meta/type_meta.rs
+++ b/rust/fory-core/src/meta/type_meta.rs
@@ -17,10 +17,13 @@
use crate::buffer::{Reader, Writer};
use crate::error::Error;
-use crate::meta::murmurhash3_x64_128;
-use crate::meta::{Encoding, MetaStringDecoder, MetaStringEncoder};
+use crate::meta::{
+ murmurhash3_x64_128, Encoding, MetaString, MetaStringDecoder, FIELD_NAME_DECODER,
+ FIELD_NAME_ENCODER, NAMESPACE_DECODER,
+};
use crate::types::TypeId;
use anyhow::anyhow;
+use std::clone::Clone;
use std::cmp::min;
const SMALL_NUM_FIELDS_THRESHOLD: usize = 0b11111;
@@ -34,7 +37,20 @@ const COMPRESS_META_FLAG: i64 = 0b1 << 13;
const HAS_FIELDS_META_FLAG: i64 = 0b1 << 12;
const NUM_HASH_BITS: i8 = 50;
-static ENCODING_OPTIONS: &[Encoding] = &[
+pub static NAMESPACE_ENCODINGS: &[Encoding] = &[
+ Encoding::Utf8,
+ Encoding::AllToLowerSpecial,
+ Encoding::LowerUpperDigitSpecial,
+];
+
+pub static TYPE_NAME_ENCODINGS: &[Encoding] = &[
+ Encoding::Utf8,
+ Encoding::AllToLowerSpecial,
+ Encoding::LowerUpperDigitSpecial,
+ Encoding::FirstToLowerSpecial,
+];
+
+static FIELD_NAME_ENCODINGS: &[Encoding] = &[
Encoding::Utf8,
Encoding::AllToLowerSpecial,
Encoding::LowerUpperDigitSpecial,
@@ -207,7 +223,7 @@ impl FieldInfo {
let field_name_bytes = reader.bytes(name_size);
- let field_name = MetaStringDecoder::new()
+ let field_name = FIELD_NAME_DECODER
.decode(field_name_bytes, encoding)
.unwrap();
FieldInfo {
@@ -220,9 +236,8 @@ impl FieldInfo {
// field_bytes: | header | type_info | field_name |
let mut writer = Writer::default();
// header: | field_name_encoding:2bits | size:4bits | nullability:1bit | ref_tracking:1bit |
- let meta_string = MetaStringEncoder::new()
- .set_options(Some(ENCODING_OPTIONS))
- .encode(&self.field_name)?;
+ let meta_string =
+ FIELD_NAME_ENCODER.encode_with_encodings(&self.field_name, FIELD_NAME_ENCODINGS)?;
let name_encoded = meta_string.bytes.as_slice();
let name_size = name_encoded.len() - 1;
let mut header: u8 = (min(FIELD_NAME_SIZE_THRESHOLD, name_size) as u8) << 2;
@@ -234,7 +249,7 @@ impl FieldInfo {
if nullable {
header |= 2;
}
- let encoding_idx = ENCODING_OPTIONS
+ let encoding_idx = FIELD_NAME_ENCODINGS
.iter()
.position(|x| *x == meta_string.encoding)
.unwrap() as u8;
@@ -259,8 +274,8 @@ impl PartialEq for FieldType {
#[derive(Debug)]
pub struct TypeMetaLayer {
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: MetaString,
+ type_name: MetaString,
register_by_name: bool,
field_infos: Vec<FieldInfo>,
}
@@ -268,8 +283,8 @@ pub struct TypeMetaLayer {
impl TypeMetaLayer {
pub fn new(
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: MetaString,
+ type_name: MetaString,
register_by_name: bool,
field_infos: Vec<FieldInfo>,
) -> TypeMetaLayer {
@@ -286,47 +301,70 @@ impl TypeMetaLayer {
self.type_id
}
+ pub fn get_type_name(&self) -> MetaString {
+ self.type_name.clone()
+ }
+
+ pub fn get_namespace(&self) -> MetaString {
+ self.namespace.clone()
+ }
+
pub fn get_field_infos(&self) -> &Vec<FieldInfo> {
&self.field_infos
}
- fn write_name(writer: &mut Writer, bytes: &[u8], encoding: Encoding) {
+ fn write_name(writer: &mut Writer, name: &MetaString, encodings: &[Encoding]) {
+ let encoding_idx = encodings.iter().position(|x| *x == name.encoding).unwrap() as u8;
+ let bytes = name.bytes.as_slice();
if bytes.len() >= BIG_NAME_THRESHOLD {
- writer.u8((BIG_NAME_THRESHOLD << 2) as u8 | encoding as u8);
+ writer.u8((BIG_NAME_THRESHOLD << 2) as u8 | encoding_idx);
writer.var_uint32((bytes.len() - BIG_NAME_THRESHOLD) as u32);
} else {
- writer.u8((bytes.len() << 2) as u8 | encoding as u8);
+ writer.u8((bytes.len() << 2) as u8 | encoding_idx);
}
writer.bytes(bytes);
}
pub fn write_namespace(&self, writer: &mut Writer) {
- Self::write_name(writer, &self.namespace, Encoding::Utf8);
+ Self::write_name(writer, &self.namespace, NAMESPACE_ENCODINGS);
}
pub fn write_type_name(&self, writer: &mut Writer) {
- Self::write_name(writer, &self.type_name, Encoding::Utf8);
+ Self::write_name(writer, &self.type_name, TYPE_NAME_ENCODINGS);
}
- fn read_name(reader: &mut Reader) -> Vec<u8> {
+ fn read_name(
+ reader: &mut Reader,
+ decoder: &MetaStringDecoder,
+ encodings: &[Encoding],
+ ) -> MetaString {
let header = reader.u8();
- let _encoding = header & 0b11;
+ let encoding_idx = header & 0b11;
let length = header >> 2;
-
let length = if length >= BIG_NAME_THRESHOLD as u8 {
BIG_NAME_THRESHOLD + reader.var_uint32() as usize
} else {
length as usize
};
- reader.bytes(length).to_vec()
+ let bytes = reader.bytes(length);
+ let encoding = encodings[encoding_idx as usize];
+ let name_str = decoder.decode(bytes, encoding).unwrap();
+ MetaString::new(
+ name_str,
+ encoding,
+ bytes.to_vec(),
+ decoder.special_char1,
+ decoder.special_char2,
+ )
+ .unwrap()
}
- pub fn read_namespace(reader: &mut Reader) -> Vec<u8> {
- Self::read_name(reader)
+ pub fn read_namespace(reader: &mut Reader) -> MetaString {
+ Self::read_name(reader, &NAMESPACE_DECODER, NAMESPACE_ENCODINGS)
}
- pub fn read_type_name(reader: &mut Reader) -> Vec<u8> {
- Self::read_name(reader)
+ pub fn read_type_name(reader: &mut Reader) -> MetaString {
+ Self::read_name(reader, &NAMESPACE_DECODER, TYPE_NAME_ENCODINGS)
}
fn to_bytes(&self) -> Result<Vec<u8>, Error> {
@@ -371,7 +409,7 @@ impl TypeMetaLayer {
type_id = TypeId::UNKNOWN as u32;
} else {
type_id = reader.var_uint32();
- let empty_name = "".as_bytes().to_vec();
+ let empty_name = MetaString::default();
namespace = empty_name.clone();
type_name = empty_name;
}
@@ -399,10 +437,18 @@ impl TypeMeta {
self.layers.first().unwrap().get_type_id()
}
+ pub fn get_type_name(&self) -> MetaString {
+ self.layers.first().unwrap().get_type_name()
+ }
+
+ pub fn get_namespace(&self) -> MetaString {
+ self.layers.first().unwrap().get_namespace()
+ }
+
pub fn from_fields(
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: MetaString,
+ type_name: MetaString,
register_by_name: bool,
field_infos: Vec<FieldInfo>,
) -> TypeMeta {
@@ -448,7 +494,7 @@ impl TypeMeta {
// global_binary_header:| hash:50bits | is_compressed:1bit | write_fields_meta:1bit | meta_size:12bits |
let meta_size = layers_writer.len() as i64;
let mut header: i64 = min(META_SIZE_MASK, meta_size);
- let write_meta_fields_flag = true;
+ let write_meta_fields_flag = self.get_field_infos().is_empty();
if write_meta_fields_flag {
header |= HAS_FIELDS_META_FLAG;
}
diff --git a/rust/fory-core/src/resolver/context.rs b/rust/fory-core/src/resolver/context.rs
index 0cfb35f78..3d4f27294 100644
--- a/rust/fory-core/src/resolver/context.rs
+++ b/rust/fory-core/src/resolver/context.rs
@@ -84,7 +84,7 @@ impl<'se> WriteContext<'se> {
pub struct ReadContext<'de, 'bf: 'de> {
pub reader: Reader<'bf>,
pub tags: Vec<&'de str>,
- pub fory: &'de Fory,
+ fory: &'de Fory,
pub meta_resolver: MetaReaderResolver,
}
@@ -106,7 +106,7 @@ impl<'de, 'bf: 'de> ReadContext<'de, 'bf> {
self.meta_resolver.get(type_index)
}
- pub fn load_meta(&mut self, offset: usize) {
+ pub fn load_meta(&mut self, offset: usize) -> usize {
self.meta_resolver.load(&mut Reader::new(
&self.reader.slice_after_cursor()[offset..],
))
diff --git a/rust/fory-core/src/resolver/meta_resolver.rs b/rust/fory-core/src/resolver/meta_resolver.rs
index cdd23023e..fedde3930 100644
--- a/rust/fory-core/src/resolver/meta_resolver.rs
+++ b/rust/fory-core/src/resolver/meta_resolver.rs
@@ -32,13 +32,14 @@ impl MetaReaderResolver {
unsafe { self.reading_type_defs.get_unchecked(index) }
}
- pub fn load(&mut self, reader: &mut Reader) {
+ pub fn load(&mut self, reader: &mut Reader) -> usize {
let meta_size = reader.var_uint32();
// self.reading_type_defs.reserve(meta_size as usize);
for _ in 0..meta_size {
self.reading_type_defs
.push(Rc::new(TypeMeta::from_bytes(reader)));
}
+ reader.get_cursor()
}
}
diff --git a/rust/fory-core/src/resolver/type_resolver.rs b/rust/fory-core/src/resolver/type_resolver.rs
index 03bfafea7..d7af1d809 100644
--- a/rust/fory-core/src/resolver/type_resolver.rs
+++ b/rust/fory-core/src/resolver/type_resolver.rs
@@ -22,58 +22,55 @@ use crate::serializer::StructSerializer;
use std::cell::RefCell;
use std::{any::Any, collections::HashMap};
+type SerializerFn = fn(&dyn Any, &mut WriteContext, is_field: bool);
+type DeserializerFn =
+ fn(&mut ReadContext, is_field: bool, skip_ref_flag: bool) -> Result<Box<dyn Any>, Error>;
+
pub struct Harness {
- serializer: fn(&dyn Any, &mut WriteContext),
- deserializer: fn(&mut ReadContext) -> Result<Box<dyn Any>, Error>,
+ serializer: SerializerFn,
+ deserializer: DeserializerFn,
}
impl Harness {
- pub fn new(
- serializer: fn(&dyn Any, &mut WriteContext),
- deserializer: fn(&mut ReadContext) -> Result<Box<dyn Any>, Error>,
- ) -> Harness {
+ pub fn new(serializer: SerializerFn, deserializer: DeserializerFn) -> Harness {
Harness {
serializer,
deserializer,
}
}
- pub fn get_serializer(&self) -> fn(&dyn Any, &mut WriteContext) {
+ pub fn get_serializer(&self) -> fn(&dyn Any, &mut WriteContext, is_field: bool) {
self.serializer
}
- pub fn get_deserializer(&self) -> fn(&mut ReadContext) -> Result<Box<dyn Any>, Error> {
+ pub fn get_deserializer(&self) -> DeserializerFn {
self.deserializer
}
}
-#[allow(dead_code)]
+#[derive(Clone, Debug)]
pub struct TypeInfo {
type_def: Vec<u8>,
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: String,
+ type_name: String,
+ register_by_name: bool,
}
impl TypeInfo {
pub fn new<T: StructSerializer>(
fory: &Fory,
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: &str,
+ type_name: &str,
register_by_name: bool,
) -> TypeInfo {
TypeInfo {
- type_def: T::type_def(
- fory,
- type_id,
- namespace.clone(),
- type_name.clone(),
- register_by_name,
- ),
+ type_def: T::type_def(fory, type_id, namespace, type_name, register_by_name),
type_id,
- namespace,
- type_name,
+ namespace: namespace.to_owned(),
+ type_name: type_name.to_owned(),
+ register_by_name,
}
}
@@ -89,11 +86,12 @@ impl TypeInfo {
#[derive(Default)]
pub struct TypeResolver {
serialize_map: HashMap<u32, Harness>,
+ name_serialize_map: HashMap<(String, String), Harness>,
type_id_map: HashMap<std::any::TypeId, u32>,
+ type_name_map: HashMap<std::any::TypeId, (String, String)>,
type_info_map: HashMap<std::any::TypeId, TypeInfo>,
// Fast lookup by numeric ID for common types
type_id_index: Vec<u32>,
- type_id_counter: u32,
sorted_field_names_map: RefCell<HashMap<std::any::TypeId, Vec<String>>>,
}
@@ -124,12 +122,18 @@ impl TypeResolver {
)
}
- pub fn register<T: StructSerializer>(&mut self, type_info: TypeInfo) {
- fn serializer<T2: 'static + StructSerializer>(this: &dyn Any, context: &mut WriteContext) {
+ pub fn register<T: StructSerializer>(&mut self, type_info: &TypeInfo) {
+ fn serializer<T2: 'static + StructSerializer>(
+ this: &dyn Any,
+ context: &mut WriteContext,
+ is_field: bool,
+ ) {
let this = this.downcast_ref::<T2>();
match this {
Some(v) => {
- T2::write(v, context, true);
+ let skip_ref_flag =
+ crate::serializer::get_skip_ref_flag::<T2>(context.get_fory());
+ crate::serializer::write_data(v, context, is_field, skip_ref_flag, true);
}
None => todo!(),
}
@@ -137,39 +141,53 @@ impl TypeResolver {
fn deserializer<T2: 'static + StructSerializer>(
context: &mut ReadContext,
+ is_field: bool,
+ skip_ref_flag: bool,
) -> Result<Box<dyn Any>, Error> {
- match T2::read_compatible(context) {
+ match crate::serializer::read_data::<T2>(context, is_field, skip_ref_flag, true) {
Ok(v) => Ok(Box::new(v)),
Err(e) => Err(e),
}
}
+ let rs_type_id = std::any::TypeId::of::<T>();
+ if self.type_info_map.contains_key(&rs_type_id) {
+ panic!("rs_struct:{:?} already registered", type_info.type_id);
+ }
+ self.type_info_map.insert(rs_type_id, (*type_info).clone());
+
let index = T::type_index() as usize;
if index >= self.type_id_index.len() {
self.type_id_index.resize(index + 1, NO_TYPE_ID);
}
self.type_id_index[index] = type_info.type_id;
- if self.serialize_map.contains_key(&type_info.type_id) {
- panic!("TypeId {:?} already registered", type_info.type_id);
- }
- self.type_id_map
- .insert(std::any::TypeId::of::<T>(), type_info.type_id);
- self.serialize_map.insert(
- type_info.type_id,
- Harness::new(serializer::<T>, deserializer::<T>),
- );
- self.type_info_map
- .insert(std::any::TypeId::of::<T>(), type_info);
- }
- pub fn next_type_id(&mut self) -> u32 {
- let old_type_id = self.type_id_counter;
- loop {
- self.type_id_counter += 1;
- if !self.serialize_map.contains_key(&self.type_id_counter) {
- break;
+ if type_info.register_by_name {
+ if self
+ .name_serialize_map
+ .contains_key(&(type_info.namespace.clone(), type_info.type_name.clone()))
+ {
+ panic!("TypeId {:?} already registered_by_name", type_info.type_id);
+ }
+ let namespace_bytes = type_info.namespace.clone();
+ let type_name_bytes = type_info.type_name.clone();
+ self.type_name_map.insert(
+ rs_type_id,
+ (namespace_bytes.clone(), type_name_bytes.clone()),
+ );
+ self.name_serialize_map.insert(
+ (namespace_bytes, type_name_bytes),
+ Harness::new(serializer::<T>, deserializer::<T>),
+ );
+ } else {
+ if self.serialize_map.contains_key(&type_info.type_id) {
+ panic!("TypeId {:?} already registered_by_id", type_info.type_id);
}
+ self.type_id_map.insert(rs_type_id, type_info.type_id);
+ self.serialize_map.insert(
+ type_info.type_id,
+ Harness::new(serializer::<T>, deserializer::<T>),
+ );
}
- old_type_id
}
pub fn get_harness_by_type(&self, type_id: std::any::TypeId) -> Option<&Harness> {
@@ -180,6 +198,11 @@ impl TypeResolver {
self.serialize_map.get(&id)
}
+ pub fn get_name_harness(&self, namespace: &str, type_name: &str) -> Option<&Harness> {
+ self.name_serialize_map
+ .get(&(namespace.to_owned(), type_name.to_owned()))
+ }
+
pub fn get_sorted_field_names<T: StructSerializer>(
&self,
type_id: std::any::TypeId,
diff --git a/rust/fory-core/src/serializer/bool.rs b/rust/fory-core/src/serializer/bool.rs
index d6f61ae2d..5e1e8828e 100644
--- a/rust/fory-core/src/serializer/bool.rs
+++ b/rust/fory-core/src/serializer/bool.rs
@@ -28,19 +28,25 @@ impl Serializer for bool {
mem::size_of::<i32>()
}
- fn write(&self, context: &mut WriteContext, is_field: bool) {
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
+ context.writer.u8(if *self { 1 } else { 0 });
+ }
+
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
context.writer.var_uint32(TypeId::BOOL as u32);
}
- context.writer.u8(if *self { 1 } else { 0 });
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ Ok(context.reader.u8() == 1)
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
let remote_type_id = context.reader.var_uint32();
assert_eq!(remote_type_id, TypeId::BOOL as u32);
}
- Ok(context.reader.u8() == 1)
}
fn get_type_id(_fory: &Fory) -> u32 {
diff --git a/rust/fory-core/src/serializer/collection.rs b/rust/fory-core/src/serializer/collection.rs
index 3e20f4fb5..de3e6f333 100644
--- a/rust/fory-core/src/serializer/collection.rs
+++ b/rust/fory-core/src/serializer/collection.rs
@@ -15,26 +15,23 @@
// specific language governing permissions and limitations
// under the License.
-use crate::ensure;
use crate::error::Error;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
-use crate::serializer::{deserialize, serialize, Serializer};
-use crate::types::{Mode, SIZE_OF_REF_AND_TYPE};
-use anyhow::anyhow;
+use crate::serializer::Serializer;
+use crate::types::Mode;
// const TRACKING_REF: u8 = 0b1;
pub const HAS_NULL: u8 = 0b10;
-// Whether collection elements type is not declare type.
-const NOT_DECL_ELEMENT_TYPE: u8 = 0b100;
+// Whether collection elements type is declare type.
+const DECL_ELEMENT_TYPE: u8 = 0b100;
-// Whether collection elements type different.
-// const NOT_SAME_TYPE: u8 = 0b1000;
+// Whether collection elements type same.
+pub const IS_SAME_TYPE: u8 = 0b1000;
-pub fn write_collection<'a, T: Serializer + 'a, I: IntoIterator<Item = &'a T>>(
- iter: I,
+pub fn write_collection_type_info(
context: &mut WriteContext,
is_field: bool,
collection_type_id: u32,
@@ -42,71 +39,77 @@ pub fn write_collection<'a, T: Serializer + 'a, I: IntoIterator<Item = &'a T>>(
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
context.writer.var_uint32(collection_type_id);
}
- let iter = iter.into_iter();
- let len = iter.size_hint().0;
+}
+
+pub fn write_collection<'a, T: Serializer + 'a, I: IntoIterator<Item = &'a T>>(
+ iter: I,
+ context: &mut WriteContext,
+ is_field: bool,
+) {
+ let items: Vec<&T> = iter.into_iter().collect();
+ let len = items.len();
context.writer.var_uint32(len as u32);
if len == 0 {
return;
}
- let has_null = T::is_option();
let mut header = 0;
+ let mut has_null = false;
+ if T::is_option() {
+ for item in &items {
+ if item.is_none() {
+ has_null = true;
+ break;
+ }
+ }
+ }
+ let is_same_type = true;
if has_null {
header |= HAS_NULL;
}
- header |= NOT_DECL_ELEMENT_TYPE;
+ if is_field {
+ header |= DECL_ELEMENT_TYPE;
+ }
+ if is_same_type {
+ header |= IS_SAME_TYPE;
+ }
context.writer.u8(header);
- context
- .writer
- .var_uint32(T::get_type_id(context.get_fory()));
- if !has_null {
- context
- .writer
- .reserve((<T as Serializer>::reserved_space()) * len);
- for item in iter {
- item.write(context, true);
- }
- } else {
- context
- .writer
- .reserve((<T as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE) * len);
- for item in iter {
- serialize(item, context, true);
- }
+ T::write_type_info(context, is_field);
+ // context.writer.reserve((T::reserved_space() + SIZE_OF_REF_AND_TYPE) * len);
+ for item in &items {
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<T>(context.get_fory());
+ let skip_ref_flag = is_same_type && !has_null;
+ crate::serializer::write_data(*item, context, is_field, skip_ref_flag, true);
}
- println!("bytes after write collection {:?}", context.writer.dump());
}
-pub fn read_collection<C, T>(
+pub fn read_collection_type_info(
context: &mut ReadContext,
is_field: bool,
- expected_collection_type_id: u32,
-) -> Result<C, Error>
+ collection_type_id: u32,
+) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ let remote_collection_type_id = context.reader.var_uint32();
+ assert_eq!(remote_collection_type_id, collection_type_id);
+ }
+}
+
+pub fn read_collection<C, T>(context: &mut ReadContext) -> Result<C, Error>
where
T: Serializer,
C: FromIterator<T>,
{
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_collection_type_id = context.reader.var_uint32();
- assert_eq!(remote_collection_type_id, expected_collection_type_id);
- }
let len = context.reader.var_uint32();
if len == 0 {
return Ok(C::from_iter(std::iter::empty()));
}
let header = context.reader.u8();
- let actual_elem_type_id = context.reader.var_uint32();
- let expected_elem_id = T::get_type_id(context.fory);
- ensure!(
- expected_elem_id == actual_elem_type_id,
- anyhow!("Invalid field type, expected:{expected_elem_id}, actual:{actual_elem_type_id}")
- );
- if header & HAS_NULL == 0 {
- (0..len)
- .map(|_| T::read(context, true))
- .collect::<Result<C, Error>>()
- } else {
- (0..len)
- .map(|_| deserialize(context, true))
- .collect::<Result<C, Error>>()
- }
+ let declared = (header & DECL_ELEMENT_TYPE) != 0;
+ T::read_type_info(context, declared);
+ let has_null = (header & HAS_NULL) != 0;
+ let is_same_type = (header & IS_SAME_TYPE) != 0;
+ let skip_ref_flag = is_same_type && !has_null;
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<T>(context.get_fory());
+ (0..len)
+ .map(|_| crate::serializer::read_data(context, declared, skip_ref_flag, true))
+ .collect::<Result<C, Error>>()
}
diff --git a/rust/fory-core/src/serializer/datetime.rs b/rust/fory-core/src/serializer/datetime.rs
index 48c0b57b9..ffe89fe8a 100644
--- a/rust/fory-core/src/serializer/datetime.rs
+++ b/rust/fory-core/src/serializer/datetime.rs
@@ -27,11 +27,7 @@ use chrono::{DateTime, Days, NaiveDate, NaiveDateTime};
use std::mem;
impl Serializer for NaiveDateTime {
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_type_id = context.reader.var_uint32();
- assert_eq!(remote_type_id, TypeId::TIMESTAMP as u32);
- }
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
let micros = context.reader.i64();
let seconds = micros / 1_000_000;
let subsec_micros = (micros % 1_000_000) as u32;
@@ -43,15 +39,25 @@ impl Serializer for NaiveDateTime {
)))
}
- fn write(&self, context: &mut WriteContext, is_field: bool) {
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- context.writer.var_uint32(TypeId::TIMESTAMP as u32);
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(remote_type_id, TypeId::TIMESTAMP as u32);
}
+ }
+
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
let dt = self.and_utc();
let micros = dt.timestamp() * 1_000_000 + dt.timestamp_subsec_micros() as i64;
context.writer.i64(micros);
}
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ context.writer.var_uint32(TypeId::TIMESTAMP as u32);
+ }
+ }
+
fn reserved_space() -> usize {
mem::size_of::<u64>()
}
@@ -64,23 +70,22 @@ impl Serializer for NaiveDateTime {
impl ForyGeneralList for NaiveDateTime {}
impl Serializer for NaiveDate {
- fn write(&self, context: &mut WriteContext, is_field: bool) {
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
+ let days_since_epoch = self.signed_duration_since(EPOCH).num_days();
+ context.writer.i32(days_since_epoch as i32);
+ }
+
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
context.writer.var_uint32(TypeId::LOCAL_DATE as u32);
}
- let days_since_epoch = self.signed_duration_since(EPOCH).num_days();
- context.writer.i32(days_since_epoch as i32);
}
fn reserved_space() -> usize {
mem::size_of::<i32>()
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_type_id = context.reader.var_uint32();
- assert_eq!(remote_type_id, TypeId::LOCAL_DATE as u32);
- }
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
let days = context.reader.i32();
EPOCH
.checked_add_days(Days::new(days as u64))
@@ -89,6 +94,13 @@ impl Serializer for NaiveDate {
)))
}
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(remote_type_id, TypeId::LOCAL_DATE as u32);
+ }
+ }
+
fn get_type_id(_fory: &Fory) -> u32 {
TypeId::LOCAL_DATE as u32
}
diff --git a/rust/fory-core/src/serializer/list.rs b/rust/fory-core/src/serializer/list.rs
index cf95253b6..a7488e89a 100644
--- a/rust/fory-core/src/serializer/list.rs
+++ b/rust/fory-core/src/serializer/list.rs
@@ -23,18 +23,28 @@ use crate::serializer::Serializer;
use crate::types::{ForyGeneralList, TypeId};
use std::mem;
-use super::collection::{read_collection, write_collection};
+use super::collection::{
+ read_collection, read_collection_type_info, write_collection, write_collection_type_info,
+};
impl<T> Serializer for Vec<T>
where
T: Serializer + ForyGeneralList,
{
fn write(&self, context: &mut WriteContext, is_field: bool) {
- write_collection(self.iter(), context, is_field, TypeId::LIST as u32);
+ write_collection(self, context, is_field);
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- read_collection(context, is_field, TypeId::LIST as u32)
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
+ write_collection_type_info(context, is_field, TypeId::LIST as u32);
+ }
+
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ read_collection(context)
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ read_collection_type_info(context, is_field, TypeId::LIST as u32)
}
fn reserved_space() -> usize {
diff --git a/rust/fory-core/src/serializer/map.rs b/rust/fory-core/src/serializer/map.rs
index f374a1cdf..c349a8836 100644
--- a/rust/fory-core/src/serializer/map.rs
+++ b/rust/fory-core/src/serializer/map.rs
@@ -15,102 +15,214 @@
// specific language governing permissions and limitations
// under the License.
-use crate::ensure;
use crate::error::Error;
use crate::fory::Fory;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
use crate::serializer::Serializer;
use crate::types::{ForyGeneralList, Mode, TypeId, SIZE_OF_REF_AND_TYPE};
-use anyhow::anyhow;
use std::collections::HashMap;
use std::mem;
const MAX_CHUNK_SIZE: u8 = 255;
-impl<T1: Serializer + Eq + std::hash::Hash, T2: Serializer> Serializer for HashMap<T1, T2> {
+const TRACKING_KEY_REF: u8 = 0b1;
+pub const KEY_NULL: u8 = 0b10;
+const DECL_KEY_TYPE: u8 = 0b100;
+const TRACKING_VALUE_REF: u8 = 0b1000;
+pub const VALUE_NULL: u8 = 0b10000;
+const DECL_VALUE_TYPE: u8 = 0b100000;
+
+fn check_and_write_null<K: Serializer + Eq + std::hash::Hash, V: Serializer>(
+ context: &mut WriteContext,
+ is_field: bool,
+ key: &K,
+ value: &V,
+) -> bool {
+ if key.is_none() && value.is_none() {
+ context.writer.u8(KEY_NULL | VALUE_NULL);
+ return true;
+ }
+ if key.is_none() {
+ let mut chunk_header = KEY_NULL;
+ let skip_ref_flag;
+ if is_field {
+ skip_ref_flag = crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
+ chunk_header |= DECL_VALUE_TYPE;
+ } else {
+ skip_ref_flag = false;
+ chunk_header |= TRACKING_VALUE_REF;
+ }
+ context.writer.u8(chunk_header);
+
+ crate::serializer::write_data(value, context, is_field, skip_ref_flag, false);
+ return true;
+ }
+ if value.is_none() {
+ let mut chunk_header = VALUE_NULL;
+ let skip_ref_flag;
+ if is_field {
+ // skip_ref_flag = crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
+ skip_ref_flag = true;
+ chunk_header |= DECL_KEY_TYPE;
+ } else {
+ skip_ref_flag = false;
+ chunk_header |= TRACKING_KEY_REF;
+ }
+ context.writer.u8(chunk_header);
+ crate::serializer::write_data(key, context, is_field, skip_ref_flag, false);
+ return true;
+ }
+ false
+}
+
+fn write_chunk_size(context: &mut WriteContext, header_offset: usize, size: u8) {
+ context.writer.set_bytes(header_offset + 1, &[size]);
+}
+
+impl<K: Serializer + Eq + std::hash::Hash, V: Serializer> Serializer for HashMap<K, V> {
fn write(&self, context: &mut WriteContext, is_field: bool) {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- context.writer.var_uint32(TypeId::MAP as u32);
+ let length = self.len();
+ context.writer.var_uint32(length as u32);
+ if length == 0 {
+ return;
}
- context.writer.var_uint32(self.len() as u32);
- let reserved_space = (<T1 as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE)
+ let reserved_space = (<K as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE)
* self.len()
- + (<T2 as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE) * self.len();
+ + (<V as Serializer>::reserved_space() + SIZE_OF_REF_AND_TYPE) * self.len();
context.writer.reserve(reserved_space);
let mut header_offset = 0;
- let mut pair_counter = 0;
- let mut header_gen = false;
+ let mut pair_counter: u8 = 0;
+ let mut need_write_header = true;
+ let mut skip_key_ref_flag = false;
+ let mut skip_val_ref_flag = false;
for entry in self.iter() {
- if !header_gen {
+ let key = entry.0;
+ let value = entry.1;
+ if need_write_header {
+ if check_and_write_null(context, is_field, key, value) {
+ continue;
+ }
header_offset = context.writer.len();
- let _is_key_null = false;
- let _is_val_null = false;
- // todo
- // if T1::is_option() {}
- // if T2::is_option() {}
context.writer.i16(-1);
- context
- .writer
- .var_uint32(T1::get_type_id(context.get_fory()));
- context
- .writer
- .var_uint32(T2::get_type_id(context.get_fory()));
- let header = 0;
- context.writer.set_bytes(header_offset, &[header]);
- header_gen = true;
+ let mut chunk_header = 0;
+ if is_field {
+ chunk_header |= DECL_KEY_TYPE;
+ chunk_header |= DECL_VALUE_TYPE;
+ }
+ // skip_key_ref_flag = crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
+ skip_key_ref_flag = true;
+ // skip_val_ref_flag = crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
+ skip_val_ref_flag = true;
+ if !skip_key_ref_flag {
+ chunk_header |= TRACKING_KEY_REF;
+ }
+ if !skip_val_ref_flag {
+ chunk_header |= TRACKING_VALUE_REF;
+ }
+ K::write_type_info(context, is_field);
+ V::write_type_info(context, is_field);
+ context.writer.set_bytes(header_offset, &[chunk_header]);
+ need_write_header = false;
}
- entry.0.write(context, true);
- entry.1.write(context, true);
+ if key.is_none() || value.is_none() {
+ write_chunk_size(context, header_offset, pair_counter);
+ pair_counter = 0;
+ need_write_header = true;
+ check_and_write_null(context, is_field, key, value);
+ continue;
+ }
+ crate::serializer::write_data(key, context, is_field, skip_key_ref_flag, true);
+ crate::serializer::write_data(value, context, is_field, skip_val_ref_flag, true);
pair_counter += 1;
if pair_counter == MAX_CHUNK_SIZE {
- context.writer.set_bytes(header_offset + 1, &[pair_counter]);
+ write_chunk_size(context, header_offset, pair_counter);
pair_counter = 0;
- header_gen = false;
+ need_write_header = true;
}
}
if pair_counter > 0 {
- assert!(header_offset > 0);
- context.writer.set_bytes(header_offset + 1, &[pair_counter]);
+ write_chunk_size(context, header_offset, pair_counter);
}
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_collection_type_id = context.reader.var_uint32();
- assert_eq!(remote_collection_type_id, TypeId::MAP as u32);
- }
- let mut map = HashMap::<T1, T2>::new();
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
let len = context.reader.var_uint32();
+ let mut map = HashMap::<K, V>::with_capacity(len as usize);
+ if len == 0 {
+ return Ok(map);
+ }
let mut len_counter = 0;
loop {
if len_counter == len {
break;
}
- let _header = context.reader.u8();
+ let header = context.reader.u8();
+ if header & KEY_NULL != 0 && header & VALUE_NULL != 0 {
+ map.insert(K::default(), V::default());
+ len_counter += 1;
+ continue;
+ }
+ let key_declared = (header & DECL_KEY_TYPE) != 0;
+ let _key_tracking_ref = (header & TRACKING_KEY_REF) != 0;
+ let value_declared = (header & DECL_VALUE_TYPE) != 0;
+ let _value_tracking_ref = (header & TRACKING_VALUE_REF) != 0;
+ if header & KEY_NULL != 0 {
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
+ let skip_ref_flag = if value_declared {
+ crate::serializer::get_skip_ref_flag::<V>(context.get_fory())
+ } else {
+ false
+ };
+ let value =
+ crate::serializer::read_data(context, value_declared, skip_ref_flag, false)?;
+ map.insert(K::default(), value);
+ len_counter += 1;
+ continue;
+ }
+ if header & VALUE_NULL != 0 {
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
+ let skip_ref_flag = if key_declared {
+ crate::serializer::get_skip_ref_flag::<K>(context.get_fory())
+ } else {
+ false
+ };
+ let key =
+ crate::serializer::read_data(context, key_declared, skip_ref_flag, false)?;
+ map.insert(key, V::default());
+ len_counter += 1;
+ continue;
+ }
let chunk_size = context.reader.u8();
- let actual_key_id = context.reader.var_uint32();
- let expected_key_id = T1::get_type_id(context.fory);
- ensure!(
- expected_key_id == actual_key_id,
- anyhow!("Invalid field type, expected:{expected_key_id}, actual:{actual_key_id}")
- );
- let actual_val_id = context.reader.var_uint32();
- let expected_val_id = T2::get_type_id(context.fory);
- ensure!(
- expected_val_id == actual_val_id,
- anyhow!("Invalid field type, expected:{expected_val_id}, actual:{actual_val_id}")
- );
+ K::read_type_info(context, key_declared);
+ V::read_type_info(context, value_declared);
assert!(len_counter + chunk_size as u32 <= len);
for _ in (0..chunk_size).enumerate() {
- map.insert(T1::read(context, true)?, T2::read(context, true)?);
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<K>(context.get_fory());
+ let key = crate::serializer::read_data(context, key_declared, true, true)?;
+ // let skip_ref_flag = crate::serializer::get_skip_ref_flag::<V>(context.get_fory());
+ let value = crate::serializer::read_data(context, value_declared, true, true)?;
+ map.insert(key, value);
}
len_counter += chunk_size as u32;
}
Ok(map)
}
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ context.writer.var_uint32(TypeId::MAP as u32);
+ }
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ let remote_collection_type_id = context.reader.var_uint32();
+ assert_eq!(remote_collection_type_id, TypeId::MAP as u32);
+ }
+ }
+
fn reserved_space() -> usize {
mem::size_of::<i32>()
}
diff --git a/rust/fory-core/src/serializer/mod.rs b/rust/fory-core/src/serializer/mod.rs
index b918db5f5..bcf207787 100644
--- a/rust/fory-core/src/serializer/mod.rs
+++ b/rust/fory-core/src/serializer/mod.rs
@@ -18,15 +18,14 @@
use crate::error::Error;
use crate::fory::Fory;
use crate::resolver::context::{ReadContext, WriteContext};
-use crate::types::RefFlag;
-use anyhow::anyhow;
+use crate::types::{Mode, RefFlag, PRIMITIVE_TYPES};
mod any;
mod bool;
pub mod collection;
mod datetime;
mod list;
-mod map;
+pub mod map;
mod number;
mod option;
mod primitive_list;
@@ -34,32 +33,57 @@ mod set;
pub mod skip;
mod string;
-pub fn serialize<T: Serializer + 'static>(record: &T, context: &mut WriteContext, is_field: bool) {
+pub fn write_data<T: Serializer + 'static>(
+ record: &T,
+ context: &mut WriteContext,
+ is_field: bool,
+ skip_ref_flag: bool,
+ skip_type_info: bool,
+) {
if record.is_none() {
context.writer.i8(RefFlag::Null as i8);
} else {
- context.writer.i8(RefFlag::NotNullValue as i8);
+ if !skip_ref_flag {
+ context.writer.i8(RefFlag::NotNullValue as i8);
+ }
+ if !skip_type_info {
+ T::write_type_info(context, is_field);
+ }
record.write(context, is_field);
}
}
-pub fn deserialize<T: Serializer + Default>(
+pub fn read_data<T: Serializer + Default>(
context: &mut ReadContext,
is_field: bool,
+ skip_ref_flag: bool,
+ skip_type_info: bool,
) -> Result<T, Error> {
- let ref_flag = context.reader.i8();
- if ref_flag == (RefFlag::NotNullValue as i8) || ref_flag == (RefFlag::RefValue as i8) {
- T::read(context, is_field)
- } else if ref_flag == (RefFlag::Null as i8) {
- Ok(T::default())
- // Err(anyhow!("Try to deserialize non-option type to null"))?
- } else if ref_flag == (RefFlag::Ref as i8) {
- Err(Error::Ref)
+ if !skip_ref_flag {
+ let ref_flag = context.reader.i8();
+ if ref_flag == RefFlag::Null as i8 {
+ Ok(T::default())
+ } else if ref_flag == (RefFlag::NotNullValue as i8) {
+ if !skip_type_info {
+ T::read_type_info(context, is_field);
+ }
+ T::read(context)
+ } else {
+ unimplemented!()
+ }
} else {
- Err(anyhow!("Unknown ref flag, value:{ref_flag}"))?
+ if !skip_type_info {
+ T::read_type_info(context, is_field);
+ }
+ T::read(context)
}
}
+pub fn get_skip_ref_flag<T: Serializer>(fory: &Fory) -> bool {
+ let elem_type_id = T::get_type_id(fory);
+ !T::is_option() && PRIMITIVE_TYPES.contains(&elem_type_id)
+}
+
pub trait Serializer
where
Self: Sized + Default + 'static,
@@ -71,18 +95,22 @@ where
/// Write the data into the buffer.
fn write(&self, context: &mut WriteContext, is_field: bool);
+ fn write_type_info(context: &mut WriteContext, is_field: bool);
+
/// Entry point of the serialization.
///
/// Step 1: write the type flag and type flag into the buffer.
/// Step 2: invoke the write function to write the Rust object.
fn serialize(&self, context: &mut WriteContext, is_field: bool) {
- serialize(self, context, is_field);
+ write_data(self, context, is_field, false, false);
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error>;
+ fn read(context: &mut ReadContext) -> Result<Self, Error>;
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool);
fn deserialize(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- deserialize(context, is_field)
+ read_data(context, is_field, false, false)
}
fn get_type_id(_fory: &Fory) -> u32;
@@ -100,13 +128,13 @@ pub trait StructSerializer: Serializer + 'static {
fn type_def(
fory: &Fory,
type_id: u32,
- namespace: Vec<u8>,
- type_name: Vec<u8>,
+ namespace: &str,
+ type_name: &str,
register_by_name: bool,
) -> Vec<u8>;
fn type_index() -> u32;
- fn actual_type_id(type_id: u32) -> u32;
+ fn actual_type_id(type_id: u32, register_by_name: bool, mode: &Mode) -> u32;
fn read_compatible(context: &mut ReadContext) -> Result<Self, Error>;
diff --git a/rust/fory-core/src/serializer/number.rs b/rust/fory-core/src/serializer/number.rs
index aa5688dc6..fc1aecb1a 100644
--- a/rust/fory-core/src/serializer/number.rs
+++ b/rust/fory-core/src/serializer/number.rs
@@ -25,19 +25,25 @@ use crate::types::{ForyGeneralList, TypeId};
macro_rules! impl_num_serializer {
($name: ident, $ty:tt, $field_type: expr) => {
impl Serializer for $ty {
- fn write(&self, context: &mut WriteContext, is_field: bool) {
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
+ context.writer.$name(*self);
+ }
+
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
if *context.get_fory().get_mode() == crate::types::Mode::Compatible && !is_field {
context.writer.var_uint32($field_type as u32);
}
- context.writer.$name(*self);
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ Ok(context.reader.$name())
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
if *context.get_fory().get_mode() == crate::types::Mode::Compatible && !is_field {
let remote_type_id = context.reader.var_uint32();
assert_eq!(remote_type_id, $field_type as u32);
}
- Ok(context.reader.$name())
}
fn reserved_space() -> usize {
diff --git a/rust/fory-core/src/serializer/option.rs b/rust/fory-core/src/serializer/option.rs
index 8a1129d0c..488682dfd 100644
--- a/rust/fory-core/src/serializer/option.rs
+++ b/rust/fory-core/src/serializer/option.rs
@@ -20,21 +20,18 @@ use crate::fory::Fory;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
use crate::serializer::Serializer;
-use crate::types::{ForyGeneralList, Mode, TypeId};
+use crate::types::ForyGeneralList;
impl<T: Serializer> Serializer for Option<T> {
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_type_id = context.reader.var_uint32();
- assert_eq!(remote_type_id, T::get_type_id(context.fory));
- }
- Ok(Some(T::read(context, is_field)?))
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ Ok(Some(T::read(context)?))
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ T::read_type_info(context, is_field);
}
fn write(&self, context: &mut WriteContext, is_field: bool) {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- context.writer.var_uint32(TypeId::BOOL as u32);
- }
if let Some(v) = self {
T::write(v, context, is_field)
} else {
@@ -42,6 +39,10 @@ impl<T: Serializer> Serializer for Option<T> {
}
}
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
+ T::write_type_info(context, is_field);
+ }
+
fn reserved_space() -> usize {
std::mem::size_of::<T>()
}
diff --git a/rust/fory-core/src/serializer/primitive_list.rs b/rust/fory-core/src/serializer/primitive_list.rs
index 4abea1a53..bfabd7e9a 100644
--- a/rust/fory-core/src/serializer/primitive_list.rs
+++ b/rust/fory-core/src/serializer/primitive_list.rs
@@ -25,10 +25,7 @@ use crate::types::TypeId;
macro_rules! impl_primitive_vec {
($name:ident, $ty:ty, $field_type:expr) => {
impl Serializer for Vec<$ty> {
- fn write(&self, context: &mut WriteContext, is_field: bool) {
- if *context.get_fory().get_mode() == crate::types::Mode::Compatible && !is_field {
- context.writer.var_uint32($field_type as u32);
- }
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
let len_bytes = self.len() * std::mem::size_of::<$ty>();
context.writer.var_uint32(len_bytes as u32);
context.writer.reserve(len_bytes);
@@ -42,31 +39,35 @@ macro_rules! impl_primitive_vec {
}
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
if *context.get_fory().get_mode() == crate::types::Mode::Compatible && !is_field {
- let remote_type_id = context.reader.var_uint32();
- assert_eq!(remote_type_id, $field_type as u32);
+ context.writer.var_uint32($field_type as u32);
}
- let size_bytes = context.reader.var_uint32() as usize;
+ }
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ let size_bytes = context.reader.var_uint32() as usize;
if size_bytes % std::mem::size_of::<$ty>() != 0 {
panic!("Invalid data length");
}
-
let len = size_bytes / std::mem::size_of::<$ty>();
-
let mut vec: Vec<$ty> = Vec::with_capacity(len);
-
unsafe {
let dst_ptr = vec.as_mut_ptr() as *mut u8;
let src = context.reader.bytes(size_bytes);
std::ptr::copy_nonoverlapping(src.as_ptr(), dst_ptr, size_bytes);
vec.set_len(len);
}
-
Ok(vec)
}
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ if *context.get_fory().get_mode() == crate::types::Mode::Compatible && !is_field {
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(remote_type_id, $field_type as u32);
+ }
+ }
+
fn reserved_space() -> usize {
std::mem::size_of::<$ty>()
}
diff --git a/rust/fory-core/src/serializer/set.rs b/rust/fory-core/src/serializer/set.rs
index 8b2949fd5..a9ba775bd 100644
--- a/rust/fory-core/src/serializer/set.rs
+++ b/rust/fory-core/src/serializer/set.rs
@@ -19,7 +19,9 @@ use crate::error::Error;
use crate::fory::Fory;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
-use crate::serializer::collection::{read_collection, write_collection};
+use crate::serializer::collection::{
+ read_collection, read_collection_type_info, write_collection, write_collection_type_info,
+};
use crate::serializer::Serializer;
use crate::types::{ForyGeneralList, TypeId};
use std::collections::HashSet;
@@ -27,11 +29,19 @@ use std::mem;
impl<T: Serializer + Eq + std::hash::Hash> Serializer for HashSet<T> {
fn write(&self, context: &mut WriteContext, is_field: bool) {
- write_collection(self.iter(), context, is_field, TypeId::SET as u32);
+ write_collection(self, context, is_field);
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
- read_collection(context, is_field, TypeId::SET as u32)
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
+ write_collection_type_info(context, is_field, TypeId::SET as u32);
+ }
+
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
+ read_collection(context)
+ }
+
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ read_collection_type_info(context, is_field, TypeId::SET as u32)
}
fn reserved_space() -> usize {
diff --git a/rust/fory-core/src/serializer/skip.rs b/rust/fory-core/src/serializer/skip.rs
index 0972afcb8..ca09fee26 100644
--- a/rust/fory-core/src/serializer/skip.rs
+++ b/rust/fory-core/src/serializer/skip.rs
@@ -18,16 +18,22 @@
use crate::error::Error;
use crate::meta::NullableFieldType;
use crate::resolver::context::ReadContext;
-use crate::serializer::collection::HAS_NULL;
+use crate::serializer::collection::{HAS_NULL, IS_SAME_TYPE};
use crate::serializer::Serializer;
-use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES};
+use crate::types::{RefFlag, TypeId, BASIC_TYPES, CONTAINER_TYPES, PRIMITIVE_TYPES};
use chrono::{NaiveDate, NaiveDateTime};
+pub fn get_read_ref_flag(field_type: &NullableFieldType) -> bool {
+ let nullable = field_type.nullable;
+ nullable || !PRIMITIVE_TYPES.contains(&field_type.type_id)
+}
+
macro_rules! basic_type_deserialize {
($tid:expr, $context:expr; $(($ty:ty, $id:ident)),+ $(,)?) => {
$(
if $tid == TypeId::$id {
- <$ty>::read($context, true)?;
+ <$ty>::read_type_info($context, true);
+ <$ty>::read($context)?;
return Ok(());
}
)+else {
@@ -45,10 +51,7 @@ pub fn skip_field_value(
) -> Result<(), Error> {
if read_ref_flag {
let ref_flag = context.reader.i8();
- if field_type.nullable
- && ref_flag != (RefFlag::NotNullValue as i8)
- && ref_flag != (RefFlag::RefValue as i8)
- {
+ if field_type.nullable && ref_flag == (RefFlag::Null as i8) {
return Ok(());
}
}
@@ -79,50 +82,102 @@ pub fn skip_field_value(
} else if CONTAINER_TYPES.contains(&type_id) {
if type_id == TypeId::LIST || type_id == TypeId::SET {
let length = context.reader.var_uint32() as usize;
- // todo
+ if length == 0 {
+ return Ok(());
+ }
let header = context.reader.u8();
- let read_ref_flag = (header & HAS_NULL) != 0;
- let _elem_type = context.reader.var_uint32();
+ let has_null = (header & HAS_NULL) != 0;
+ let is_same_type = (header & IS_SAME_TYPE) != 0;
+ let skip_ref_flag = is_same_type && !has_null;
+ let elem_type = field_type.generics.first().unwrap();
for _ in 0..length {
- skip_field_value(
- context,
- field_type.generics.first().unwrap(),
- read_ref_flag,
- )?;
+ skip_field_value(context, elem_type, !skip_ref_flag)?;
}
} else if type_id == TypeId::MAP {
- todo!();
- let length = context.reader.var_uint32() as usize;
- for _ in 0..length {
- skip_field_value(context, field_type.generics.first().unwrap(), true)?;
- skip_field_value(context, field_type.generics.get(1).unwrap(), true)?;
+ let length = context.reader.var_uint32();
+ if length == 0 {
+ return Ok(());
+ }
+ let mut len_counter = 0;
+ let key_type = field_type.generics.first().unwrap();
+ let value_type = field_type.generics.get(1).unwrap();
+ loop {
+ if len_counter == length {
+ break;
+ }
+ let header = context.reader.u8();
+ if header & crate::serializer::map::KEY_NULL != 0
+ && header & crate::serializer::map::VALUE_NULL != 0
+ {
+ len_counter += 1;
+ continue;
+ }
+ if header & crate::serializer::map::KEY_NULL != 0 {
+ // let read_ref_flag = get_read_ref_flag(value_type);
+ skip_field_value(context, value_type, false).unwrap();
+ len_counter += 1;
+ continue;
+ }
+ if header & crate::serializer::map::VALUE_NULL != 0 {
+ // let read_ref_flag = get_read_ref_flag(key_type);
+ skip_field_value(context, key_type, false).unwrap();
+ len_counter += 1;
+ continue;
+ }
+ let chunk_size = context.reader.u8();
+ for _ in (0..chunk_size).enumerate() {
+ // let read_ref_flag = get_read_ref_flag(key_type);
+ skip_field_value(context, key_type, false).unwrap();
+ // let read_ref_flag = get_read_ref_flag(value_type);
+ skip_field_value(context, value_type, false).unwrap();
+ }
+ len_counter += chunk_size as u32;
}
}
Ok(())
+ } else if type_id == TypeId::NAMED_ENUM {
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(type_id_num, remote_type_id);
+ let meta_index = context.reader.var_uint32();
+ let type_def = context.get_meta(meta_index as usize);
+ let type_resolver = context.get_fory().get_type_resolver();
+ type_resolver
+ .get_name_harness(
+ &type_def.get_namespace().original,
+ &type_def.get_type_name().original,
+ )
+ .unwrap()
+ .get_deserializer()(context, true, true)?;
+ Ok(())
} else {
- unreachable!()
+ unreachable!();
}
}
Err(_) => {
- let tag = type_id_num & 0xff;
- if tag == TypeId::COMPATIBLE_STRUCT as u32 {
+ let internal_id = type_id_num & 0xff;
+ const COMPATIBLE_STRUCT_ID: u32 = TypeId::COMPATIBLE_STRUCT as u32;
+ const NAMED_COMPATIBLE_STRUCT_ID: u32 = TypeId::NAMED_COMPATIBLE_STRUCT as u32;
+ const ENUM_ID: u32 = TypeId::ENUM as u32;
+ if internal_id == COMPATIBLE_STRUCT_ID || internal_id == NAMED_COMPATIBLE_STRUCT_ID {
let remote_type_id = context.reader.var_uint32();
let meta_index = context.reader.var_uint32();
let type_def = context.get_meta(meta_index as usize);
assert_eq!(remote_type_id, type_def.get_type_id());
- let field_infos: Vec<_> = type_def.get_field_infos().to_vec();
+ let field_infos = type_def.get_field_infos().to_vec();
for field_info in field_infos.iter() {
let nullable_field_type =
NullableFieldType::from(field_info.field_type.clone());
- skip_field_value(context, &nullable_field_type, true)?;
+ let read_ref_flag = get_read_ref_flag(&nullable_field_type);
+ skip_field_value(context, &nullable_field_type, read_ref_flag)?;
}
- } else if tag == TypeId::ENUM as u32 {
- context
- .fory
- .get_type_resolver()
+ } else if internal_id == ENUM_ID {
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(type_id_num, remote_type_id);
+ let type_resolver = context.get_fory().get_type_resolver();
+ type_resolver
.get_harness(type_id_num)
.unwrap()
- .get_deserializer()(context)?;
+ .get_deserializer()(context, true, true)?;
} else {
unimplemented!()
}
diff --git a/rust/fory-core/src/serializer/string.rs b/rust/fory-core/src/serializer/string.rs
index bc1bf1592..d92096a46 100644
--- a/rust/fory-core/src/serializer/string.rs
+++ b/rust/fory-core/src/serializer/string.rs
@@ -35,28 +35,42 @@ impl Serializer for String {
mem::size_of::<i32>()
}
- fn write(&self, context: &mut WriteContext, is_field: bool) {
- if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- context.writer.var_uint32(TypeId::STRING as u32);
- }
+ fn write(&self, context: &mut WriteContext, _is_field: bool) {
let mut len = get_latin1_length(self);
if len >= 0 {
let bitor = (len as u64) << 2 | StrEncoding::Latin1 as u64;
context.writer.var_uint36_small(bitor);
context.writer.latin1_string(self);
- } else {
+ } else if context.get_fory().is_compress_string() {
+ // todo: support `writeNumUtf16BytesForUtf8Encoding` like in java
len = self.len() as i32;
let bitor = (len as u64) << 2 | StrEncoding::Utf8 as u64;
context.writer.var_uint36_small(bitor);
context.writer.utf8_string(self);
+ } else {
+ let utf16: Vec<u16> = self.encode_utf16().collect();
+ let bitor = (utf16.len() as u64 * 2) << 2 | StrEncoding::Utf16 as u64;
+ context.writer.var_uint36_small(bitor);
+ for unit in utf16 {
+ #[cfg(target_endian = "little")]
+ {
+ context.writer.u16(unit);
+ }
+ #[cfg(target_endian = "big")]
+ {
+ unimplemented!()
+ }
+ }
}
}
- fn read(context: &mut ReadContext, is_field: bool) -> Result<Self, Error> {
+ fn write_type_info(context: &mut WriteContext, is_field: bool) {
if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
- let remote_type_id = context.reader.var_uint32();
- assert_eq!(remote_type_id, TypeId::STRING as u32);
+ context.writer.var_uint32(TypeId::STRING as u32);
}
+ }
+
+ fn read(context: &mut ReadContext) -> Result<Self, Error> {
let bitor = context.reader.var_uint36_small();
let len = bitor >> 2;
let encoding = bitor & 0b11;
@@ -76,6 +90,13 @@ impl Serializer for String {
Ok(s)
}
+ fn read_type_info(context: &mut ReadContext, is_field: bool) {
+ if *context.get_fory().get_mode() == Mode::Compatible && !is_field {
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(remote_type_id, TypeId::STRING as u32);
+ }
+ }
+
fn get_type_id(_fory: &Fory) -> u32 {
TypeId::STRING as u32
}
diff --git a/rust/fory-core/src/types.rs b/rust/fory-core/src/types.rs
index 39a395296..389395327 100644
--- a/rust/fory-core/src/types.rs
+++ b/rust/fory-core/src/types.rs
@@ -59,7 +59,7 @@ pub enum TypeId {
NAMED_ENUM = 14,
STRUCT = 15,
COMPATIBLE_STRUCT = 16,
- NamedStruct = 17,
+ NAMED_STRUCT = 17,
NAMED_COMPATIBLE_STRUCT = 18,
EXT = 19,
NAMED_EXT = 20,
@@ -126,6 +126,16 @@ pub static BASIC_TYPES: [TypeId; 18] = [
TypeId::FLOAT64_ARRAY,
];
+pub static PRIMITIVE_TYPES: [u32; 7] = [
+ TypeId::BOOL as u32,
+ TypeId::INT8 as u32,
+ TypeId::INT16 as u32,
+ TypeId::INT32 as u32,
+ TypeId::INT64 as u32,
+ TypeId::FLOAT32 as u32,
+ TypeId::FLOAT64 as u32,
+];
+
pub static FINAL_TYPES: [TypeId; 3] = [TypeId::STRING, TypeId::LOCAL_DATE, TypeId::TIMESTAMP];
pub static PRIMITIVE_ARRAY_TYPES: [TypeId; 8] = [
diff --git a/rust/fory-derive/src/object/derive_enum.rs b/rust/fory-derive/src/object/derive_enum.rs
index 6c9efa703..7adf1c661 100644
--- a/rust/fory-derive/src/object/derive_enum.rs
+++ b/rust/fory-derive/src/object/derive_enum.rs
@@ -21,7 +21,16 @@ use syn::DataEnum;
pub fn gen_type_def(_data_enum: &DataEnum) -> TokenStream {
quote! {
- Vec::new()
+ let namespace_metastring = fory_core::meta::NAMESPACE_ENCODER.encode_with_encodings(namespace, fory_core::meta::NAMESPACE_ENCODINGS).unwrap();
+ let type_name_metastring = fory_core::meta::TYPE_NAME_ENCODER.encode_with_encodings(type_name, fory_core::meta::TYPE_NAME_ENCODINGS).unwrap();
+ let meta = fory_core::meta::TypeMeta::from_fields(
+ type_id,
+ namespace_metastring,
+ type_name_metastring,
+ register_by_name,
+ vec![],
+ );
+ meta.to_bytes().unwrap()
}
}
@@ -30,11 +39,13 @@ pub fn gen_write(data_enum: &DataEnum) -> TokenStream {
let variant_values: Vec<_> = (0..variant_idents.len()).map(|v| v as u32).collect();
quote! {
- fn write(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) {
- if !is_field {
- let type_id = Self::get_type_id(context.get_fory());
- context.writer.var_uint32(type_id);
- }
+ fn serialize(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) {
+ context.writer.i8(fory_core::types::RefFlag::NotNullValue as i8);
+ Self::write_type_info(context, is_field);
+ self.write(context, is_field);
+ }
+
+ fn write(&self, context: &mut fory_core::resolver::context::WriteContext, _is_field: bool) {
match self {
#(
Self::#variant_idents => {
@@ -44,6 +55,19 @@ pub fn gen_write(data_enum: &DataEnum) -> TokenStream {
}
}
+ fn write_type_info(context: &mut fory_core::resolver::context::WriteContext, is_field: bool){
+ if *context.get_fory().get_mode() == fory_core::types::Mode::Compatible {
+ let type_id = Self::get_type_id(context.get_fory());
+ context.writer.var_uint32(type_id);
+ if type_id & 0xff == fory_core::types::TypeId::NAMED_ENUM as u32 {
+ let meta_index = context.push_meta(
+ std::any::TypeId::of::<Self>()
+ ) as u32;
+ context.writer.var_uint32(meta_index);
+ }
+ }
+ }
+
fn reserved_space() -> usize {
4
}
@@ -55,15 +79,21 @@ pub fn gen_read(data_enum: &DataEnum) -> TokenStream {
let variant_values: Vec<_> = (0..variant_idents.len()).map(|v| v as u32).collect();
quote! {
+ fn deserialize(context: &mut fory_core::resolver::context::ReadContext, _is_field: bool) -> Result<Self, fory_core::error::Error> {
+ let ref_flag = context.reader.i8();
+ if ref_flag == fory_core::types::RefFlag::Null as i8 {
+ Ok(Self::default())
+ } else if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) {
+ Self::read_type_info(context, false);
+ Self::read(context)
+ } else {
+ unimplemented!()
+ }
+ }
+
fn read(
context: &mut fory_core::resolver::context::ReadContext,
- is_field: bool
) -> Result<Self, fory_core::error::Error> {
- if !is_field {
- let remote_type_id = context.reader.var_uint32();
- let local_type_id = Self::get_type_id(context.get_fory());
- assert_eq!(remote_type_id, local_type_id);
- }
let ordinal = context.reader.var_uint32();
match ordinal {
#(
@@ -72,19 +102,35 @@ pub fn gen_read(data_enum: &DataEnum) -> TokenStream {
_ => panic!("unknown value"),
}
}
+
+ fn read_type_info(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) {
+ if *context.get_fory().get_mode() == fory_core::types::Mode::Compatible {
+ let local_type_id = Self::get_type_id(context.get_fory());
+ let remote_type_id = context.reader.var_uint32();
+ assert_eq!(local_type_id, remote_type_id);
+ if local_type_id & 0xff == fory_core::types::TypeId::NAMED_ENUM as u32 {
+ let _meta_index = context.reader.var_uint32();
+ }
+ }
+ }
}
}
pub fn gen_actual_type_id() -> TokenStream {
quote! {
- (type_id << 8) + fory_core::types::TypeId::ENUM as u32
+ if register_by_name {
+ fory_core::types::TypeId::NAMED_ENUM as u32
+ } else {
+ (type_id << 8) + fory_core::types::TypeId::ENUM as u32
+ }
}
}
pub fn gen_read_compatible(_data_enum: &DataEnum) -> TokenStream {
quote! {
fn read_compatible(context: &mut fory_core::resolver::context::ReadContext) -> Result<Self, fory_core::error::Error> {
- <Self as fory_core::serializer::Serializer>::read(context, true)
+ <Self as fory_core::serializer::Serializer>::read_type_info(context, true);
+ <Self as fory_core::serializer::Serializer>::read(context)
}
}
}
diff --git a/rust/fory-derive/src/object/misc.rs b/rust/fory-derive/src/object/misc.rs
index ec7eac6bf..a23afce4a 100644
--- a/rust/fory-derive/src/object/misc.rs
+++ b/rust/fory-derive/src/object/misc.rs
@@ -66,7 +66,7 @@ fn type_def(fields: &[&Field]) -> TokenStream {
});
quote! {
let sorted_field_names = <Self as fory_core::serializer::StructSerializer>::get_sorted_field_names(fory);
- let field_infos = vec![#(#field_infos),*];
+ let field_infos: Vec<fory_core::meta::FieldInfo> = vec![#(#field_infos),*];
let mut sorted_field_infos = Vec::with_capacity(field_infos.len());
for name in &sorted_field_names {
if let Some(info) = field_infos.iter().find(|f| &f.field_name == name) {
@@ -75,10 +75,12 @@ fn type_def(fields: &[&Field]) -> TokenStream {
panic!("Field {} not found in field_infos", name);
}
}
+ let namespace_metastring = fory_core::meta::NAMESPACE_ENCODER.encode_with_encodings(namespace, fory_core::meta::NAMESPACE_ENCODINGS).unwrap();
+ let type_name_metastring = fory_core::meta::TYPE_NAME_ENCODER.encode_with_encodings(type_name, fory_core::meta::TYPE_NAME_ENCODINGS).unwrap();
let meta = fory_core::meta::TypeMeta::from_fields(
type_id,
- namespace,
- type_name,
+ namespace_metastring,
+ type_name_metastring,
register_by_name,
sorted_field_infos,
);
@@ -97,7 +99,20 @@ pub fn gen_in_struct_impl(fields: &[&Field]) -> TokenStream {
pub fn gen_actual_type_id() -> TokenStream {
quote! {
- (type_id << 8) + fory_core::types::TypeId::COMPATIBLE_STRUCT as u32
+ if mode == &fory_core::types::Mode::Compatible {
+ if register_by_name {
+ fory_core::types::TypeId::NAMED_COMPATIBLE_STRUCT as u32
+ } else {
+ (type_id << 8) + fory_core::types::TypeId::COMPATIBLE_STRUCT as u32
+ }
+ } else {
+ if register_by_name {
+ fory_core::types::TypeId::NAMED_STRUCT as u32
+ } else {
+ (type_id << 8) + fory_core::types::TypeId::STRUCT as u32
+ }
+ }
+
}
}
diff --git a/rust/fory-derive/src/object/read.rs b/rust/fory-derive/src/object/read.rs
index 0a1833aba..fae9ad747 100644
--- a/rust/fory-derive/src/object/read.rs
+++ b/rust/fory-derive/src/object/read.rs
@@ -67,7 +67,9 @@ fn read(fields: &[&Field]) -> TokenStream {
.iter()
.map(|f| create_private_field_name(f))
.collect();
- let sorted_deserialize = {
+ let sorted_deserialize = if fields.is_empty() {
+ quote! {}
+ } else {
let let_decls = fields
.iter()
.zip(private_idents.iter())
@@ -77,20 +79,18 @@ fn read(fields: &[&Field]) -> TokenStream {
let mut #private_ident: #ty = Default::default();
}
});
-
let match_ts = fields.iter().zip(private_idents.iter()).map(|(field, private_ident)| {
let ty = &field.ty;
let name_str = field.ident.as_ref().unwrap().to_string();
quote! {
#name_str => {
- #private_ident = <#ty as fory_core::serializer::Serializer>::deserialize(context, true)?;
+ let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory());
+ #private_ident = fory_core::serializer::read_data::<#ty>(context, true, skip_ref_flag, false)?;
}
}
});
-
quote! {
#(#let_decls)*
-
let sorted_field_names = <Self as fory_core::serializer::StructSerializer>::get_sorted_field_names(context.get_fory());
for field_name in sorted_field_names {
match field_name.as_str() {
@@ -110,19 +110,22 @@ fn read(fields: &[&Field]) -> TokenStream {
}
});
quote! {
- fn read(context: &mut fory_core::resolver::context::ReadContext, _is_field: bool) -> Result<Self, fory_core::error::Error> {
+ fn read(context: &mut fory_core::resolver::context::ReadContext) -> Result<Self, fory_core::error::Error> {
+ #sorted_deserialize
+ Ok(Self {
+ #(#field_idents),*
+ })
+ // Ok(Self {
+ // #(#assign_stmt),*
+ // })
+ }
+
+ fn read_type_info(context: &mut fory_core::resolver::context::ReadContext, _is_field: bool) {
let remote_type_id = context.reader.var_uint32();
assert_eq!(remote_type_id, Self::get_type_id(context.get_fory()));
if *context.get_fory().get_mode() == fory_core::types::Mode::Compatible {
let _meta_index = context.reader.var_uint32();
}
- // Ok(Self {
- // #(#assign_stmt),*
- // })
- #sorted_deserialize
- Ok(Self {
- #(#field_idents),*
- })
}
}
}
@@ -151,7 +154,7 @@ pub fn gen_deserialize_nullable(fields: &[&Field]) -> TokenStream {
let ty = &field.ty;
let generic_tree = parse_generic_tree(ty);
let nullable_generic_tree = NullableTypeNode::from(generic_tree);
- let deserialize_tokens = nullable_generic_tree.to_deserialize_tokens(&vec![]);
+ let deserialize_tokens = nullable_generic_tree.to_deserialize_tokens(&vec![], true);
quote! {
fn #fn_name(
context: &mut fory_core::resolver::context::ReadContext,
@@ -191,7 +194,8 @@ pub fn gen_read_compatible(fields: &[&Field], struct_ident: &Ident) -> TokenStre
if _field.field_name.as_str() == #field_name_str {
let local_field_type = #generic_token;
if &_field.field_type == &local_field_type {
- #var_name = Some(<#ty as fory_core::serializer::Serializer>::deserialize(context, true).unwrap_or_else(|_err| {
+ let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory());
+ #var_name = Some(fory_core::serializer::read_data::<#ty>(context, true, skip_ref_flag, false).unwrap_or_else(|_err| {
// same type, err means something wrong
panic!("Err at deserializing {:?}: {:?}", #field_name_str, _err);
}));
@@ -201,7 +205,8 @@ pub fn gen_read_compatible(fields: &[&Field], struct_ident: &Ident) -> TokenStre
if local_nullable_type != remote_nullable_type {
// set default and skip bytes
println!("Type not match, just skip: {}", #field_name_str);
- fory_core::serializer::skip::skip_field_value(context, &remote_nullable_type, true).unwrap();
+ let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&remote_nullable_type);
+ fory_core::serializer::skip::skip_field_value(context, &remote_nullable_type, read_ref_flag).unwrap();
#var_name = Some(#base_ty::default());
} else {
println!("Try to deserialize: {}", #field_name_str);
@@ -236,7 +241,8 @@ pub fn gen_read_compatible(fields: &[&Field], struct_ident: &Ident) -> TokenStre
#(#pattern_items else)* {
println!("skip {:?}:{:?}", _field.field_name.as_str(), _field.field_type);
let nullable_field_type = fory_core::meta::NullableFieldType::from(_field.field_type.clone());
- fory_core::serializer::skip::skip_field_value(context, &nullable_field_type, true).unwrap();
+ let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(&nullable_field_type);
+ fory_core::serializer::skip::skip_field_value(context, &nullable_field_type, read_ref_flag).unwrap();
}
}
Ok(Self {
@@ -251,10 +257,18 @@ pub fn gen(fields: &[&Field], struct_ident: &Ident) -> TokenStream {
let compatible_token_stream = deserialize_compatible(struct_ident);
quote! {
- fn deserialize(context: &mut fory_core::resolver::context::ReadContext, is_field: bool) -> Result<Self, fory_core::error::Error> {
+ fn deserialize(context: &mut fory_core::resolver::context::ReadContext, _is_field: bool) -> Result<Self, fory_core::error::Error> {
match context.get_fory().get_mode() {
fory_core::types::Mode::SchemaConsistent => {
- fory_core::serializer::deserialize::<Self>(context, is_field)
+ let ref_flag = context.reader.i8();
+ if ref_flag == fory_core::types::RefFlag::Null as i8 {
+ Ok(Self::default())
+ } else if ref_flag == (fory_core::types::RefFlag::NotNullValue as i8) {
+ Self::read_type_info(context, false);
+ Self::read(context)
+ } else {
+ unimplemented!()
+ }
},
fory_core::types::Mode::Compatible => {
#compatible_token_stream
diff --git a/rust/fory-derive/src/object/serializer.rs b/rust/fory-derive/src/object/serializer.rs
index fc5e7497d..4a8c42713 100644
--- a/rust/fory-derive/src/object/serializer.rs
+++ b/rust/fory-derive/src/object/serializer.rs
@@ -66,10 +66,10 @@ pub fn derive_serializer(ast: &syn::DeriveInput) -> TokenStream {
let gen = quote! {
impl fory_core::serializer::StructSerializer for #name {
- fn type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: Vec<u8>, type_name: Vec<u8>, register_by_name: bool) -> Vec<u8> {
+ fn type_def(fory: &fory_core::fory::Fory, type_id: u32, namespace: &str, type_name: &str, register_by_name: bool) -> Vec<u8> {
#type_def_token_stream
}
- fn actual_type_id(type_id: u32) -> u32 {
+ fn actual_type_id(type_id: u32, register_by_name: bool, mode: &fory_core::types::Mode) -> u32 {
#actual_type_id_token_stream
}
#type_index_token_stream
diff --git a/rust/fory-derive/src/object/util.rs b/rust/fory-derive/src/object/util.rs
index 2491600ee..b511b6728 100644
--- a/rust/fory-derive/src/object/util.rs
+++ b/rust/fory-derive/src/object/util.rs
@@ -41,22 +41,16 @@ macro_rules! basic_type_deserialize {
$ty_str => {
if $nullable {
quote! {
- let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
- None
- } else {
- Some(<$ty as fory_core::serializer::Serializer>::read(context, true)
- .map_err(fory_core::error::Error::from)?)
- };
+ <$ty as fory_core::serializer::Serializer>::read_type_info(context, true);
+ let res1 = Some(<$ty as fory_core::serializer::Serializer>::read(context)
+ .map_err(fory_core::error::Error::from)?);
Ok::<Option<$ty>, fory_core::error::Error>(res1)
}
} else {
quote! {
- let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
- $ty::default()
- } else {
- <$ty as fory_core::serializer::Serializer>::read(context, true)
- .map_err(fory_core::error::Error::from)?
- };
+ <$ty as fory_core::serializer::Serializer>::read_type_info(context, true);
+ let res2 = <$ty as fory_core::serializer::Serializer>::read(context)
+ .map_err(fory_core::error::Error::from)?;
Ok::<$ty, fory_core::error::Error>(res2)
}
}
@@ -114,7 +108,11 @@ pub fn try_primitive_vec_type_name(node: &NullableTypeNode) -> Option<String> {
}
impl NullableTypeNode {
- pub(super) fn to_deserialize_tokens(&self, generic_path: &Vec<i8>) -> TokenStream {
+ pub(super) fn to_deserialize_tokens(
+ &self,
+ generic_path: &Vec<i8>,
+ read_ref_flag: bool,
+ ) -> TokenStream {
let tokens = if let Some(primitive_ty_name) = try_primitive_vec_type_name(self) {
let ty_type: Type = parse_str(&primitive_ty_name).expect("Invalid primitive type name");
let nullable = self.nullable;
@@ -123,7 +121,8 @@ impl NullableTypeNode {
let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
None
} else {
- Some(<#ty_type as fory_core::serializer::Serializer>::read(context, true)
+ <#ty_type as fory_core::serializer::Serializer>::read_type_info(context, true);
+ Some(<#ty_type as fory_core::serializer::Serializer>::read(context)
.map_err(fory_core::error::Error::from)?)
};
Ok::<Option<#ty_type>, fory_core::error::Error>(res1)
@@ -133,7 +132,8 @@ impl NullableTypeNode {
let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
Vec::default()
} else {
- <#ty_type as fory_core::serializer::Serializer>::read(context, true)
+ <#ty_type as fory_core::serializer::Serializer>::read_type_info(context, true);
+ <#ty_type as fory_core::serializer::Serializer>::read(context)
.map_err(fory_core::error::Error::from)?
};
Ok::<#ty_type, fory_core::error::Error>(res2)
@@ -158,12 +158,41 @@ impl NullableTypeNode {
match self.name.as_str() {
"Vec" => {
new_path.push(0);
+ let generic_node = self.generics.first().unwrap();
+ let element_tokens = generic_node.to_deserialize_tokens(&new_path, false);
+ let element_ty: Type = parse_str(&generic_node.to_string()).unwrap();
+ let vec_ts = quote! {
+ let length = context.reader.var_uint32() as usize;
+ if length == 0 {
+ Vec::default()
+ } else {
+ let mut v = Vec::with_capacity(length);
+ let header = context.reader.u8();
+ let has_null = (header & fory_core::serializer::collection::HAS_NULL) != 0;
+ let is_same_type = (header & fory_core::serializer::collection::IS_SAME_TYPE) != 0;
+ let read_ref_flag = !(is_same_type && !has_null);
+ for _ in 0..length {
+ let ref_flag = if read_ref_flag {
+ context.reader.i8()
+ } else {
+ fory_core::types::RefFlag::NotNullValue as i8
+ };
+ let element = if ref_flag == fory_core::types::RefFlag::Null as i8 {
+ <#element_ty>::default()
+ } else {
+ #element_tokens?
+ };
+ v.push(element);
+ }
+ v
+ }
+ };
if self.nullable {
quote! {
let v = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
None
} else {
- Some(fory_core::serializer::collection::read_collection(context, true, fory_core::types::TypeId::LIST as u32)?)
+ Some({#vec_ts})
};
Ok::<#ty, fory_core::error::Error>(v)
}
@@ -172,7 +201,7 @@ impl NullableTypeNode {
let v = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
Vec::default()
} else {
- fory_core::serializer::collection::read_collection(context, true, fory_core::types::TypeId::LIST as u32)?
+ #vec_ts
};
Ok::<#ty, fory_core::error::Error>(v)
}
@@ -180,12 +209,41 @@ impl NullableTypeNode {
}
"HashSet" => {
new_path.push(0);
+ let generic_node = self.generics.first().unwrap();
+ let element_tokens = generic_node.to_deserialize_tokens(&new_path, false);
+ let element_ty: Type = parse_str(&generic_node.to_string()).unwrap();
+ let set_ts = quote! {
+ let length = context.reader.var_uint32() as usize;
+ if length == 0 {
+ HashSet::default()
+ } else {
+ let mut s = HashSet::with_capacity(length);
+ let header = context.reader.u8();
+ let has_null = (header & fory_core::serializer::collection::HAS_NULL) != 0;
+ let is_same_type = (header & fory_core::serializer::collection::IS_SAME_TYPE) != 0;
+ let read_ref_flag = !(is_same_type && !has_null);
+ for _ in 0..length {
+ let ref_flag = if read_ref_flag {
+ context.reader.i8()
+ } else {
+ fory_core::types::RefFlag::NotNullValue as i8
+ };
+ let element = if ref_flag == fory_core::types::RefFlag::Null as i8 {
+ <#element_ty>::default()
+ } else {
+ #element_tokens?
+ };
+ s.insert(element);
+ }
+ s
+ }
+ };
if self.nullable {
quote! {
let s = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
None
} else {
- Some(fory_core::serializer::collection::read_collection(context, true, fory_core::types::TypeId::SET as u32)?)
+ Some({#set_ts})
};
Ok::<#ty, fory_core::error::Error>(s)
}
@@ -194,7 +252,7 @@ impl NullableTypeNode {
let s = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
HashSet::default()
} else {
- fory_core::serializer::collection::read_collection(context, true, fory_core::types::TypeId::SET as u32)?
+ #set_ts
};
Ok::<#ty, fory_core::error::Error>(s)
}
@@ -203,17 +261,64 @@ impl NullableTypeNode {
"HashMap" => {
let key_generic_node = self.generics.first().unwrap();
let val_generic_node = self.generics.get(1).unwrap();
+
new_path.push(0);
+ let key_tokens = key_generic_node.to_deserialize_tokens(&new_path, false);
new_path.pop();
new_path.push(1);
+ let val_tokens = val_generic_node.to_deserialize_tokens(&new_path, false);
+
let key_ty: Type = parse_str(&key_generic_node.to_string()).unwrap();
let val_ty: Type = parse_str(&val_generic_node.to_string()).unwrap();
+
+ let map_ts = quote! {
+ let length = context.reader.var_uint32();
+ let mut map = HashMap::with_capacity(length as usize);
+ if length == 0 {
+ map
+ } else {
+ let mut len_counter = 0;
+ loop {
+ if len_counter == length {
+ break;
+ }
+ let header = context.reader.u8();
+ if header & fory_core::serializer::map::KEY_NULL != 0 && header & fory_core::serializer::map::VALUE_NULL != 0 {
+ map.insert(<#key_ty>::default(), <#val_ty>::default());
+ len_counter += 1;
+ continue;
+ }
+ if header & fory_core::serializer::map::KEY_NULL != 0 {
+ let value: #val_ty = {#val_tokens}?;
+ map.insert(<#key_ty>::default(), value);
+ len_counter += 1;
+ continue;
+ }
+ if header & fory_core::serializer::map::VALUE_NULL != 0 {
+ let key: #key_ty = {#key_tokens}?;
+ map.insert(key, <#val_ty>::default());
+ len_counter += 1;
+ continue;
+ }
+ let chunk_size = context.reader.u8();
+ <#key_ty as fory_core::serializer::Serializer>::read_type_info(context, true);
+ <#val_ty as fory_core::serializer::Serializer>::read_type_info(context, true);
+ for _ in (0..chunk_size).enumerate() {
+ let key: #key_ty = {#key_tokens}?;
+ let value: #val_ty = {#val_tokens}?;
+ map.insert(key, value);
+ }
+ len_counter += chunk_size as u32;
+ }
+ map
+ }
+ };
if self.nullable {
quote! {
let m = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
None
} else {
- Some(<HashMap<#key_ty, #val_ty> as fory_core::serializer::Serializer>::read(context, true)?)
+ Some({#map_ts})
};
Ok::<#ty, fory_core::error::Error>(m)
}
@@ -222,7 +327,7 @@ impl NullableTypeNode {
let m = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
HashMap::default()
} else {
- <HashMap<#key_ty, #val_ty> as fory_core::serializer::Serializer>::read(context, true)?
+ #map_ts
};
Ok::<#ty, fory_core::error::Error>(m)
}
@@ -234,40 +339,51 @@ impl NullableTypeNode {
// struct or enum
let nullable_ty = parse_str::<Type>(&self.nullable_ty_string()).unwrap();
let ty = parse_str::<Type>(&self.to_string()).unwrap();
- if self.nullable {
+ let ts = if self.nullable {
quote! {
- const COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::COMPATIBLE_STRUCT as u32;
- const ENUM_ID: u32 = fory_core::types::TypeId::ENUM as u32;
let res1 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
None
} else {
- Some(<#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context).map_err(fory_core::error::Error::from)?)
+ let type_id = cur_remote_nullable_type.type_id;
+ let internal_id = type_id & 0xff;
+ Some(if internal_id == COMPATIBLE_STRUCT_ID || internal_id == NAMED_COMPATIBLE_STRUCT_ID {
+ <#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context)
+ .map_err(fory_core::error::Error::from)?
+ } else if internal_id == ENUM_ID || internal_id == NAMED_ENUM_ID {
+ <#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context)
+ .map_err(fory_core::error::Error::from)?
+ } else {
+ unimplemented!()
+ })
};
Ok::<#ty, fory_core::error::Error>(res1)
}
} else {
quote! {
- const COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::COMPATIBLE_STRUCT as u32;
- const ENUM_ID: u32 = fory_core::types::TypeId::ENUM as u32;
let res2 = if cur_remote_nullable_type.nullable && ref_flag == (fory_core::types::RefFlag::Null as i8) {
#ty::default()
} else {
let type_id = cur_remote_nullable_type.type_id;
let internal_id = type_id & 0xff;
- match internal_id {
- COMPATIBLE_STRUCT_ID => {
- <#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context)
+ if internal_id == COMPATIBLE_STRUCT_ID || internal_id == NAMED_COMPATIBLE_STRUCT_ID {
+ <#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context)
.map_err(fory_core::error::Error::from)?
- }
- ENUM_ID => {
- <#nullable_ty as fory_core::serializer::Serializer>::read(context, true)
+ } else if internal_id == ENUM_ID || internal_id == NAMED_ENUM_ID {
+ <#nullable_ty as fory_core::serializer::StructSerializer>::read_compatible(context)
.map_err(fory_core::error::Error::from)?
- }
- _ => unimplemented!(),
+ } else {
+ unimplemented!("")
}
};
Ok::<#ty, fory_core::error::Error>(res2)
}
+ };
+ quote! {
+ const COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::COMPATIBLE_STRUCT as u32;
+ const ENUM_ID: u32 = fory_core::types::TypeId::ENUM as u32;
+ const NAMED_COMPATIBLE_STRUCT_ID: u32 = fory_core::types::TypeId::NAMED_COMPATIBLE_STRUCT as u32;
+ const NAMED_ENUM_ID: u32 = fory_core::types::TypeId::NAMED_ENUM as u32;
+ #ts
}
};
let mut cur_remote_nullable_type = quote! { remote_nullable_type };
@@ -276,10 +392,26 @@ impl NullableTypeNode {
#cur_remote_nullable_type.generics.get(#idx as usize).unwrap()
};
}
+ let read_ref_flag_ts = if read_ref_flag {
+ quote! {
+ let read_ref_flag = fory_core::serializer::skip::get_read_ref_flag(cur_remote_nullable_type);
+ let ref_flag = if read_ref_flag {
+ context.reader.i8()
+ } else {
+ fory_core::types::RefFlag::NotNullValue as i8
+ };
+ if ref_flag == fory_core::types::RefFlag::Null as i8 {
+ Ok(Default::default())
+ } else {
+ #tokens
+ }
+ }
+ } else {
+ quote! { #tokens }
+ };
quote! {
let cur_remote_nullable_type = &#cur_remote_nullable_type;
- let ref_flag = context.reader.i8();
- #tokens
+ #read_ref_flag_ts
}
}
@@ -422,7 +554,7 @@ pub(super) fn generic_tree_to_tokens(node: &TypeNode, have_context: bool) -> Tok
let ty: syn::Type = syn::parse_str(&node.to_string()).unwrap();
let param = if have_context {
quote! {
- context.fory
+ context.get_fory()
}
} else {
quote! {
@@ -715,12 +847,12 @@ pub(super) fn get_sort_fields_ts(fields: &[&Field]) -> TokenStream {
quote! {
let field_type_id = <#ty_type as fory_core::serializer::Serializer>::get_type_id(fory);
let internal_id = field_type_id & 0xff;
- if internal_id == fory_core::types::TypeId::COMPATIBLE_STRUCT as u32 {
+ if internal_id == fory_core::types::TypeId::COMPATIBLE_STRUCT as u32 || internal_id == fory_core::types::TypeId::NAMED_COMPATIBLE_STRUCT as u32 || internal_id == fory_core::types::TypeId::STRUCT as u32 || internal_id == fory_core::types::TypeId::NAMED_STRUCT as u32 {
other_fields.push((field_type_id, #name.to_string()));
- } else if internal_id == fory_core::types::TypeId::ENUM as u32 {
+ } else if internal_id == fory_core::types::TypeId::ENUM as u32 || internal_id == fory_core::types::TypeId::NAMED_ENUM as u32 {
final_fields.push((field_type_id, #name.to_string()));
} else {
- unimplemented!();
+ unimplemented!("unknown internal_id when group_sort_enum_other_fields");
}
}
})
diff --git a/rust/fory-derive/src/object/write.rs b/rust/fory-derive/src/object/write.rs
index 4851f420a..59e7f4342 100644
--- a/rust/fory-derive/src/object/write.rs
+++ b/rust/fory-derive/src/object/write.rs
@@ -20,14 +20,17 @@ use quote::quote;
use syn::Field;
pub fn gen(fields: &[&Field]) -> TokenStream {
- let sorted_serialize = {
+ let sorted_serialize = if fields.is_empty() {
+ quote! {}
+ } else {
let match_ts = fields.iter().map(|field| {
let ty = &field.ty;
let ident = &field.ident;
let name_str = ident.as_ref().unwrap().to_string();
quote! {
#name_str => {
- <#ty as fory_core::serializer::Serializer>::serialize(&self.#ident, context, true);
+ let skip_ref_flag = fory_core::serializer::get_skip_ref_flag::<#ty>(context.get_fory());
+ fory_core::serializer::write_data::<#ty>(&self.#ident, context, true, skip_ref_flag, false);
}
}
});
@@ -66,18 +69,29 @@ pub fn gen(fields: &[&Field]) -> TokenStream {
};
quote! {
- fn serialize(&self, context: &mut fory_core::resolver::context::WriteContext, is_field: bool) {
+ fn serialize(&self, context: &mut fory_core::resolver::context::WriteContext, _is_field: bool) {
match context.get_fory().get_mode() {
fory_core::types::Mode::SchemaConsistent => {
- fory_core::serializer::serialize(self, context, is_field);
+ context.writer.i8(fory_core::types::RefFlag::NotNullValue as i8);
+ Self::write_type_info(context, false);
+ self.write(context, true);
},
fory_core::types::Mode::Compatible => {
- fory_core::serializer::serialize(self, context, is_field);
+ context.writer.i8(fory_core::types::RefFlag::NotNullValue as i8);
+ Self::write_type_info(context, false);
+ self.write(context, true);
}
}
}
fn write(&self, context: &mut fory_core::resolver::context::WriteContext, _is_field: bool) {
+ // write way before
+ // #(#accessor_expr)*
+ // sort and write
+ #sorted_serialize
+ }
+
+ fn write_type_info(context: &mut fory_core::resolver::context::WriteContext, _is_field: bool){
let type_id = Self::get_type_id(context.get_fory());
context.writer.var_uint32(type_id);
if *context.get_fory().get_mode() == fory_core::types::Mode::Compatible {
@@ -86,11 +100,6 @@ pub fn gen(fields: &[&Field]) -> TokenStream {
) as u32;
context.writer.var_uint32(meta_index);
}
- // write fields
- // write way before
- // #(#accessor_expr)*
- // sort and write
- #sorted_serialize
}
fn reserved_space() -> usize {
diff --git a/rust/tests/tests/test_compatible.rs b/rust/tests/tests/test_compatible.rs
index 8178d861d..d0e20fa5a 100644
--- a/rust/tests/tests/test_compatible.rs
+++ b/rust/tests/tests/test_compatible.rs
@@ -18,17 +18,16 @@
use fory_core::fory::Fory;
use fory_core::types::Mode::Compatible;
use fory_derive::Fory;
-use std::collections::HashSet;
+use std::collections::{HashMap, HashSet};
// RUSTFLAGS="-Awarnings" cargo expand -p fory-tests --test test_compatible
#[test]
fn simple() {
#[derive(Fory, Debug, Default)]
struct Animal1 {
- // f1: HashMap<i8, Vec<i8>>,
+ f1: HashMap<i8, Vec<i8>>,
f2: String,
f3: Vec<i8>,
- // f4: String,
f5: String,
f6: Vec<i8>,
f7: i8,
@@ -37,8 +36,7 @@ fn simple() {
#[derive(Fory, Debug, Default)]
struct Animal2 {
- // f1: HashMap<i8, Vec<i8>>,
- // f2: String,
+ f1: HashMap<i8, Vec<i8>>,
f3: Vec<i8>,
f4: String,
f5: i8,
@@ -51,7 +49,7 @@ fn simple() {
fory1.register::<Animal1>(999);
fory2.register::<Animal2>(999);
let animal: Animal1 = Animal1 {
- // f1: HashMap::from([(1, vec![2])]),
+ f1: HashMap::from([(1, vec![2])]),
f2: String::from("hello"),
f3: vec![1, 2, 3],
f5: String::from("f5"),
@@ -61,7 +59,7 @@ fn simple() {
};
let bin = fory1.serialize(&animal);
let obj: Animal2 = fory2.deserialize(&bin).unwrap();
- // assert_eq!(animal.f1, obj.f1);
+ assert_eq!(animal.f1, obj.f1);
assert_eq!(animal.f3, obj.f3);
assert_eq!(obj.f4, String::default());
assert_eq!(obj.f5, i8::default());
@@ -230,11 +228,11 @@ fn nullable_container() {
f2: Option<Vec<i8>>,
f3: HashSet<i8>,
f4: Option<HashSet<i8>>,
- // f5: HashMap<i8, Vec<i8>>,
- // f6: Option<HashMap<i8, Vec<i8>>>,
+ f5: HashMap<i8, Vec<i8>>,
+ f6: Option<HashMap<i8, Vec<i8>>>,
f7: Option<Vec<i8>>,
f8: Option<HashSet<i8>>,
- // f9: Option<HashMap<i8, i8>>,
+ f9: Option<HashMap<i8, i8>>,
last: i64,
}
@@ -244,11 +242,11 @@ fn nullable_container() {
f2: Vec<i8>,
f3: Option<HashSet<i8>>,
f4: HashSet<i8>,
- // f5: Option<HashMap<i8, Vec<i8>>>,
- // f6: HashMap<i8, Vec<i8>>,
+ f5: Option<HashMap<i8, Vec<i8>>>,
+ f6: HashMap<i8, Vec<i8>>,
f7: Vec<i8>,
f8: HashSet<i8>,
- // f9: HashMap<i8, i8>,
+ f9: HashMap<i8, i8>,
last: i64,
}
@@ -262,11 +260,11 @@ fn nullable_container() {
f2: Some(vec![43]),
f3: HashSet::from([44, 45]),
f4: Some(HashSet::from([46, 47])),
- // f5: HashMap::from([(48, vec![49])]),
- // f6: Some(HashMap::from([(48, vec![49])])),
+ f5: HashMap::from([(48, vec![49])]),
+ f6: Some(HashMap::from([(48, vec![49])])),
f7: None,
f8: None,
- // f9: None,
+ f9: None,
last: 666,
};
@@ -277,11 +275,11 @@ fn nullable_container() {
assert_eq!(item2.f2, item1.f2.unwrap());
assert_eq!(item2.f3.unwrap(), item1.f3);
assert_eq!(item2.f4, item1.f4.unwrap());
- // assert_eq!(item2.f5.unwrap(), item1.f5);
- // assert_eq!(item2.f6, item1.f6.unwrap());
+ assert_eq!(item2.f5.unwrap(), item1.f5);
+ assert_eq!(item2.f6, item1.f6.unwrap());
assert_eq!(item2.f7, Vec::default());
assert_eq!(item2.f8, HashSet::default());
- // assert_eq!(item2.f9, HashMap::default());
+ assert_eq!(item2.f9, HashMap::default());
assert_eq!(item2.last, item1.last);
}
@@ -291,14 +289,16 @@ fn inner_nullable() {
pub struct Item1 {
f1: Vec<Option<String>>,
f2: HashSet<Option<i8>>,
- // f3: HashMap<i8, Option<i8>>,
+ f3: HashMap<i8, Option<i8>>,
+ last: i64,
}
#[derive(Fory, Debug, Default)]
pub struct Item2 {
f1: Vec<String>,
f2: HashSet<i8>,
- // f3: HashMap<i8, i8>,
+ f3: HashMap<i8, i8>,
+ last: i64,
}
let mut fory1 = Fory::default().mode(Compatible);
let mut fory2 = Fory::default().mode(Compatible);
@@ -308,14 +308,16 @@ fn inner_nullable() {
let item1 = Item1 {
f1: vec![None, Some("hello".to_string())],
f2: HashSet::from([None, Some(43)]),
- // f3: HashMap::from([(44, None), (45, Some(46))]),
+ f3: HashMap::from([(44, None), (45, Some(46))]),
+ last: 666,
};
let bin = fory1.serialize(&item1);
let item2: Item2 = fory2.deserialize(&bin).unwrap();
assert_eq!(item2.f1, vec![String::default(), "hello".to_string()]);
assert_eq!(item2.f2, HashSet::from([0, 43]));
- // assert_eq!(item2.f3, HashMap::from([(44, 0), (45, 46)]));
+ assert_eq!(item2.f3, HashMap::from([(44, 0), (45, 46)]));
+ assert_eq!(item2.last, item1.last);
}
#[test]
@@ -357,8 +359,8 @@ fn nullable_struct() {
},
f2: None,
f3: Some(Item {
- name: "f3".to_string(),
- data: vec![None, Some("hello".to_string())],
+ name: "b".to_string(),
+ data: vec![None, Some("a".to_string())],
last: 45,
}),
last: 46,
@@ -436,7 +438,6 @@ fn enum_without_payload() {
last: 10,
};
let bin = fory1.serialize(&person1);
- println!("bin: {:?}", bin);
let person2: Person2 = fory2.deserialize(&bin).expect("");
assert_eq!(person2.f1, person1.f1);
assert_eq!(person2.f2, Color2::default());
@@ -446,3 +447,145 @@ fn enum_without_payload() {
assert_eq!(person2.f8.unwrap(), person1.f8);
assert_eq!(person2.last, person1.last);
}
+
+#[test]
+fn named_enum() {
+ #[derive(Fory, Debug, PartialEq, Default)]
+ enum Color {
+ #[default]
+ Green,
+ Red,
+ Blue,
+ White,
+ }
+ #[derive(Fory, Debug, PartialEq, Default)]
+ struct Item1 {
+ f1: Color,
+ f2: Color,
+ f3: Option<Color>,
+ f4: Option<Color>,
+ f5: Option<Color>,
+ f6: Option<Color>,
+ // skip
+ f7: Color,
+ f8: Option<Color>,
+ f9: Option<Color>,
+ last: i8,
+ }
+ #[derive(Fory, Debug, PartialEq, Default)]
+ struct Item2 {
+ f1: Color,
+ f2: Option<Color>,
+ f3: Color,
+ f4: Option<Color>,
+ f5: Color,
+ f6: Option<Color>,
+ last: i8,
+ }
+ let mut fory1 = Fory::default().mode(Compatible).xlang(true);
+ fory1.register_by_name::<Color>("a");
+ fory1.register::<Item1>(101);
+ let mut fory2 = Fory::default().mode(Compatible).xlang(true);
+ fory2.register_by_name::<Color>("a");
+ fory2.register::<Item2>(101);
+ let item1 = Item1 {
+ f1: Color::Red,
+ f2: Color::Blue,
+ f3: Some(Color::White),
+ f4: Some(Color::White),
+ f5: None,
+ f6: None,
+ f7: Color::White,
+ f8: Some(Color::White),
+ f9: Some(Color::White),
+ last: 42,
+ };
+ let expected_item2 = Item2 {
+ f1: Color::Red,
+ f2: Some(Color::Blue),
+ f3: Color::White,
+ f4: Some(Color::White),
+ f5: Color::default(),
+ f6: None,
+ last: 42,
+ };
+ let bin = fory1.serialize(&item1);
+ let actual_item2: Item2 = fory2.deserialize(&bin).unwrap();
+ assert_eq!(expected_item2, actual_item2);
+}
+
+#[test]
+#[allow(clippy::unnecessary_literal_unwrap)]
+fn boxed() {
+ #[derive(Fory, Debug, PartialEq, Default)]
+ struct Item1 {
+ f1: i32,
+ f2: i32,
+ f3: Option<i32>,
+ f4: Option<i32>,
+ f5: Option<i32>,
+ f6: Option<i32>,
+ }
+
+ #[derive(Fory, Debug, PartialEq, Default)]
+ struct Item2 {
+ f1: i32,
+ f2: Option<i32>,
+ f3: Option<i32>,
+ f4: i32,
+ f5: i32,
+ f6: Option<i32>,
+ }
+
+ let mut fory1 = Fory::default().mode(Compatible).xlang(true);
+ fory1.register::<Item1>(101);
+ let mut fory2 = Fory::default().mode(Compatible).xlang(true);
+ fory2.register::<Item2>(101);
+
+ let f1 = 1;
+ let f2 = 2;
+ let f3 = Some(3);
+ let f4 = Some(4);
+ let f5: Option<i32> = None;
+ let f6: Option<i32> = None;
+ let item1 = Item1 {
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ };
+ let bytes = fory1.serialize(&item1);
+ let item2: Item2 = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f1, f1);
+ assert_eq!(item2.f2.unwrap(), f2);
+ assert_eq!(item2.f3, f3);
+ assert_eq!(item2.f4, f4.unwrap());
+ assert_eq!(item2.f5, i32::default());
+ assert_eq!(item2.f6, f6);
+
+ let bytes = fory1.serialize(&f1);
+ let item2_f1: i32 = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f1, item2_f1);
+
+ let bytes = fory1.serialize(&f2);
+ let item2_f2: Option<i32> = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f2, item2_f2);
+
+ let bytes = fory1.serialize(&f3);
+ let item2_f3: Option<i32> = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f3, item2_f3);
+
+ let bytes = fory1.serialize(&f4);
+ let item2_f4: i32 = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f4, item2_f4);
+
+ let bytes = fory1.serialize(&f5);
+ let item2_f5: i32 = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f5, item2_f5);
+
+ let bytes = fory1.serialize(&f6);
+ let item2_f6: Option<i32> = fory2.deserialize(&bytes).unwrap();
+ assert_eq!(item2.f6, item2_f6);
+}
diff --git a/rust/tests/tests/test_cross_language.rs b/rust/tests/tests/test_cross_language.rs
index c348f7027..05b7fbd97 100644
--- a/rust/tests/tests/test_cross_language.rs
+++ b/rust/tests/tests/test_cross_language.rs
@@ -48,13 +48,15 @@ struct Item {
#[derive(Fory, Debug, PartialEq, Default)]
struct SimpleStruct {
// field_order != sorted_order
- // f1: HashMap<i32, f64>,
- // f2: i32,
+ f1: HashMap<i32, f64>,
+ f2: i32,
f3: Item,
f4: Option<String>,
- f5: Color,
- // f6: Vec<Option<String>>,
- // last: i32
+ // f5: Color,
+ f6: Vec<Option<String>>,
+ f7: i32,
+ // f8: i32,
+ // last: i32,
}
#[test]
@@ -219,8 +221,17 @@ fn test_string_serializer() {
let data_file_path = get_data_file();
let bytes = fs::read(&data_file_path).unwrap();
let reader = Reader::new(bytes.as_slice());
- let fory = Fory::default().mode(Compatible).xlang(true);
+ let fory = Fory::default()
+ .mode(Compatible)
+ .xlang(true)
+ .compress_string(false);
let mut context = ReadContext::new(&fory, reader);
+ let reader_compress = Reader::new(bytes.as_slice());
+ let fory_compress = Fory::default()
+ .mode(Compatible)
+ .xlang(true)
+ .compress_string(true);
+ let mut context_compress = ReadContext::new(&fory_compress, reader_compress);
let test_strings: Vec<String> = vec![
// Latin1
"ab".to_string(),
@@ -235,7 +246,8 @@ fn test_string_serializer() {
];
for s in &test_strings {
// make is_field=true to skip read/write type_id
- assert_eq!(*s, String::read(&mut context, true).unwrap());
+ assert_eq!(*s, String::read(&mut context).unwrap());
+ assert_eq!(*s, String::read(&mut context_compress).unwrap());
}
let mut writer = Writer::default();
let fory = Fory::default().mode(Compatible).xlang(true);
@@ -257,6 +269,8 @@ macro_rules! assert_de {
#[ignore]
#[allow(deprecated)]
fn test_cross_language_serializer() {
+ let day = NaiveDate::from_ymd_opt(2021, 11, 23).unwrap();
+ let instant = NaiveDateTime::from_timestamp(100, 0);
let str_list = vec!["hello".to_string(), "world".to_string()];
let str_set = HashSet::from(["hello".to_string(), "world".to_string()]);
let str_map = HashMap::<String, String>::from([
@@ -285,18 +299,8 @@ fn test_cross_language_serializer() {
assert_de!(fory, context, f32, -1f32);
assert_de!(fory, context, f64, -1f64);
assert_de!(fory, context, String, "str".to_string());
- assert_de!(
- fory,
- context,
- NaiveDate,
- NaiveDate::from_ymd_opt(2021, 11, 23).unwrap()
- );
- assert_de!(
- fory,
- context,
- NaiveDateTime,
- NaiveDateTime::from_timestamp(100, 0)
- );
+ assert_de!(fory, context, NaiveDate, day);
+ assert_de!(fory, context, NaiveDateTime, instant);
assert_de!(fory, context, Vec<bool>, [true, false]);
assert_de!(fory, context, Vec<i16>, [1, i16::MAX]);
assert_de!(fory, context, Vec<i32>, [1, i32::MAX]);
@@ -324,11 +328,8 @@ fn test_cross_language_serializer() {
fory.serialize_with_context(&-1f32, &mut context);
fory.serialize_with_context(&-1f64, &mut context);
fory.serialize_with_context(&"str".to_string(), &mut context);
- fory.serialize_with_context(
- &NaiveDate::from_ymd_opt(2021, 11, 23).unwrap(),
- &mut context,
- );
- fory.serialize_with_context(&NaiveDateTime::from_timestamp(100, 0), &mut context);
+ fory.serialize_with_context(&day, &mut context);
+ fory.serialize_with_context(&instant, &mut context);
fory.serialize_with_context(&vec![true, false], &mut context);
fory.serialize_with_context(&vec![1, i16::MAX], &mut context);
fory.serialize_with_context(&vec![1, i32::MAX], &mut context);
@@ -351,21 +352,239 @@ fn test_simple_struct() {
fory.register::<Color>(101);
fory.register::<Item>(102);
fory.register::<SimpleStruct>(103);
+
+ let local_obj = SimpleStruct {
+ f1: HashMap::from([(1, 1.0f64), (2, 2.0f64)]),
+ f2: 39,
+ f3: Item {
+ name: Some("item".to_string()),
+ },
+ f4: Some("f4".to_string()),
+ // f5: Color::White,
+ f6: vec![Some("f6".to_string())],
+ f7: 40,
+ // f8: 41,
+ // last: 42,
+ };
let remote_obj: SimpleStruct = fory.deserialize(&bytes).unwrap();
+ assert_eq!(remote_obj, local_obj);
+ let new_bytes = fory.serialize(&remote_obj);
+ let new_local_obj: SimpleStruct = fory.deserialize(&new_bytes).unwrap();
+ assert_eq!(new_local_obj, local_obj);
+ fs::write(&data_file_path, new_bytes).unwrap();
+}
+
+#[test]
+#[ignore]
+fn test_simple_named_struct() {
+ let data_file_path = get_data_file();
+ let bytes = fs::read(&data_file_path).unwrap();
+ let mut fory = Fory::default().mode(Compatible).xlang(true);
+ fory.register_by_namespace::<Color>("demo", "color");
+ fory.register_by_namespace::<Item>("demo", "item");
+ fory.register_by_namespace::<SimpleStruct>("demo", "simple_struct");
+
let local_obj = SimpleStruct {
- // f1: HashMap::from([(1, 1.0f64), (2, 2.0f64)]),
- // f2: 10,
+ f1: HashMap::from([(1, 1.0f64), (2, 2.0f64)]),
+ f2: 39,
f3: Item {
name: Some("item".to_string()),
},
- f4: Some("f3".to_string()),
- f5: Color::White,
- // f6: vec![Some("f6".to_string())],
+ f4: Some("f4".to_string()),
+ // f5: Color::White,
+ f6: vec![Some("f6".to_string())],
+ f7: 40,
+ // f8: 41,
// last: 42,
};
+ let remote_obj: SimpleStruct = fory.deserialize(&bytes).unwrap();
assert_eq!(remote_obj, local_obj);
let new_bytes = fory.serialize(&remote_obj);
- let new_local_obj: SimpleStruct = fory.deserialize(new_bytes.as_slice()).unwrap();
+ let new_local_obj: SimpleStruct = fory.deserialize(&new_bytes).unwrap();
assert_eq!(new_local_obj, local_obj);
fs::write(&data_file_path, new_bytes).unwrap();
}
+
+#[test]
+#[ignore]
+fn test_list() {
+ let data_file_path = get_data_file();
+ let bytes = fs::read(&data_file_path).unwrap();
+
+ let mut fory = Fory::default().mode(Compatible);
+ fory.register::<Item>(102);
+ let reader = Reader::new(bytes.as_slice());
+
+ let str_list = vec![Some("a".to_string()), Some("b".to_string())];
+ let str_list2 = vec![None, Some("b".to_string())];
+ let item = Item {
+ name: Some("a".to_string()),
+ };
+ let item2 = Item {
+ name: Some("b".to_string()),
+ };
+ let item3 = Item {
+ name: Some("c".to_string()),
+ };
+ let item_list = vec![Some(item), Some(item2)];
+ let item_list2 = vec![None, Some(item3)];
+
+ let bytes = reader.slice();
+ let remote_str_list: Vec<Option<String>> = fory.deserialize(bytes).unwrap();
+ assert_eq!(remote_str_list, str_list);
+ let data_bytes1 = fory.serialize(&remote_str_list);
+ let new_local_str_list: Vec<Option<String>> = fory.deserialize(&data_bytes1).unwrap();
+ assert_eq!(new_local_str_list, str_list);
+
+ let bytes = &bytes[data_bytes1.len()..];
+ let remote_str_list2: Vec<Option<String>> = fory.deserialize(bytes).unwrap();
+ assert_eq!(remote_str_list2, str_list2);
+ let data_bytes2 = fory.serialize(&remote_str_list2);
+ let new_local_str_list2: Vec<Option<String>> = fory.deserialize(&data_bytes2).unwrap();
+ assert_eq!(new_local_str_list2, str_list2);
+
+ let bytes = &bytes[data_bytes2.len()..];
+ let remote_item_list: Vec<Option<Item>> = fory.deserialize(bytes).unwrap();
+ assert_eq!(remote_item_list, item_list);
+ let data_bytes3 = fory.serialize(&remote_item_list);
+ let new_local_item_list: Vec<Option<Item>> = fory.deserialize(&data_bytes3).unwrap();
+ assert_eq!(new_local_item_list, item_list);
+
+ let bytes = &bytes[data_bytes3.len()..];
+ let remote_item_list2: Vec<Option<Item>> = fory.deserialize(bytes).unwrap();
+ assert_eq!(remote_item_list2, item_list2);
+ let data_bytes4 = fory.serialize(&remote_item_list2);
+ let new_local_item_list2: Vec<Option<Item>> = fory.deserialize(&data_bytes4).unwrap();
+ assert_eq!(new_local_item_list2, item_list2);
+
+ let all_bytes = [
+ data_bytes1.as_slice(),
+ data_bytes2.as_slice(),
+ data_bytes3.as_slice(),
+ data_bytes4.as_slice(),
+ ]
+ .concat();
+ fs::write(&data_file_path, all_bytes).unwrap();
+}
+
+#[test]
+#[ignore]
+fn test_map() {
+ let data_file_path = get_data_file();
+ let bytes = fs::read(&data_file_path).unwrap();
+
+ let mut fory = Fory::default().mode(Compatible);
+ fory.register::<Item>(102);
+ let reader = Reader::new(bytes.as_slice());
+ let mut context = ReadContext::new(&fory, reader);
+
+ let str_map = HashMap::from([
+ (Some("k1".to_string()), Some("v1".to_string())),
+ (None, Some("v2".to_string())),
+ (Some("k3".to_string()), None),
+ (Some("k4".to_string()), Some("v4".to_string())),
+ ]);
+ let item_map = HashMap::from([
+ (
+ Some("k1".to_string()),
+ Some(Item {
+ name: Some("item1".to_string()),
+ }),
+ ),
+ (
+ None,
+ Some(Item {
+ name: Some("item2".to_string()),
+ }),
+ ),
+ (Some("k3".to_string()), None),
+ (
+ Some("k4".to_string()),
+ Some(Item {
+ name: Some("item3".to_string()),
+ }),
+ ),
+ ]);
+
+ let remote_str_map: HashMap<Option<String>, Option<String>> =
+ fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_str_map, str_map);
+ let data_bytes1 = fory.serialize(&remote_str_map);
+ let new_local_str_map: HashMap<Option<String>, Option<String>> =
+ fory.deserialize(&data_bytes1).unwrap();
+ assert_eq!(new_local_str_map, str_map);
+
+ let remote_item_map: HashMap<Option<String>, Option<Item>> =
+ fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_item_map, item_map);
+ let data_bytes2 = fory.serialize(&remote_item_map);
+ let new_local_item_map: HashMap<Option<String>, Option<Item>> =
+ fory.deserialize(&data_bytes2).unwrap();
+ assert_eq!(new_local_item_map, item_map);
+
+ let all_bytes = [data_bytes1.as_slice(), data_bytes2.as_slice()].concat();
+ fs::write(&data_file_path, all_bytes).unwrap();
+}
+
+#[test]
+#[ignore]
+fn test_integer() {
+ #[derive(Fory, Debug, PartialEq, Default)]
+ struct Item2 {
+ f1: i32,
+ f2: Option<i32>,
+ f3: Option<i32>,
+ f4: i32,
+ f5: i32,
+ f6: Option<i32>,
+ }
+
+ let data_file_path = get_data_file();
+ let bytes = fs::read(&data_file_path).unwrap();
+
+ let mut fory = Fory::default().mode(Compatible);
+ fory.register::<Item2>(101);
+ let reader = Reader::new(bytes.as_slice());
+
+ let mut context = ReadContext::new(&fory, reader);
+ let f1 = 1;
+ let f2 = Some(2);
+ let f3 = Some(3);
+ let f4 = 4;
+ let f5 = i32::default();
+ let f6 = None;
+
+ let local_item2 = Item2 {
+ f1,
+ f2,
+ f3,
+ f4,
+ f5,
+ f6,
+ };
+ let remote_item2: Item2 = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_item2, local_item2);
+ let remote_f1: i32 = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f1, f1);
+ let remote_f2: Option<i32> = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f2, f2);
+ let remote_f3: Option<i32> = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f3, f3);
+ let remote_f4: i32 = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f4, f4);
+ let remote_f5: i32 = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f5, f5);
+ let remote_f6: Option<i32> = fory.deserialize_with_context(&mut context).unwrap();
+ assert_eq!(remote_f6, f6);
+
+ let mut writer = Writer::default();
+ let mut context = WriteContext::new(&fory, &mut writer);
+ // fory.serialize_with_context(&remote_item2, &mut context);
+ fory.serialize_with_context(&remote_f1, &mut context);
+ fory.serialize_with_context(&remote_f2, &mut context);
+ fory.serialize_with_context(&remote_f3, &mut context);
+ fory.serialize_with_context(&remote_f4, &mut context);
+ fory.serialize_with_context(&remote_f5, &mut context);
+ fory.serialize_with_context(&remote_f6, &mut context);
+ fs::write(&data_file_path, context.writer.dump()).unwrap();
+}
diff --git a/rust/tests/tests/test_meta_string.rs b/rust/tests/tests/test_meta_string.rs
index 701505211..1eaa4678a 100644
--- a/rust/tests/tests/test_meta_string.rs
+++ b/rust/tests/tests/test_meta_string.rs
@@ -17,11 +17,14 @@
use std::iter;
-use fory_core::meta::{Encoding, MetaStringDecoder, MetaStringEncoder};
+use fory_core::meta::{
+ Encoding, MetaStringDecoder, MetaStringEncoder, NAMESPACE_DECODER, NAMESPACE_ENCODER,
+ TYPE_NAME_DECODER, TYPE_NAME_ENCODER,
+};
#[test]
fn test_encode_meta_string_lower_special() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let bytes1 = encoder.encode_lower_special("abc_def").unwrap();
assert_eq!(bytes1.len(), 5);
let bytes2 = encoder
@@ -31,7 +34,7 @@ fn test_encode_meta_string_lower_special() {
assert_eq!(bytes2.len(), 19);
let bytes3 = encoder.encode("MediaContent").unwrap().bytes;
assert_eq!(bytes3.len(), 9);
- let decoder = MetaStringDecoder::new();
+ let decoder = &TYPE_NAME_DECODER;
assert_eq!(
decoder.decode(&bytes1, Encoding::LowerSpecial).unwrap(),
"abc_def"
@@ -49,7 +52,7 @@ fn test_encode_meta_string_lower_special() {
}
}
-fn create_string(length: usize) -> String {
+fn create_string(length: usize, special_char1: char, special_char2: char) -> String {
(0..length)
.map(|j| {
let n = j % 64;
@@ -57,8 +60,8 @@ fn create_string(length: usize) -> String {
0..=25 => (b'a' + n as u8) as char,
26..=51 => (b'A' + (n - 26) as u8) as char,
52..=61 => (b'0' + (n - 52) as u8) as char,
- 62 => '.',
- 63 => '_',
+ 62 => special_char1,
+ 63 => special_char2,
_ => unreachable!(),
}
})
@@ -67,20 +70,24 @@ fn create_string(length: usize) -> String {
#[test]
fn test_encode_meta_string_lower_upper_digit_special() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &NAMESPACE_ENCODER;
let encoded = encoder
.encode_lower_upper_digit_special("ExampleInput123")
.unwrap();
assert_eq!(encoded.len(), 12);
- let decoder = MetaStringDecoder::new();
+ let decoder = &NAMESPACE_DECODER;
let decoded = decoder
.decode(&encoded, Encoding::LowerUpperDigitSpecial)
.unwrap();
assert_eq!(decoded, "ExampleInput123");
for i in 1..128 {
- let origin_string = create_string(i);
+ let origin_string = create_string(
+ i,
+ NAMESPACE_ENCODER.special_char1,
+ NAMESPACE_ENCODER.special_char2,
+ );
let encoded = encoder
.encode_lower_upper_digit_special(&origin_string)
.unwrap();
@@ -93,27 +100,27 @@ fn test_encode_meta_string_lower_upper_digit_special() {
#[test]
fn test_meta_string() {
- let encoder = MetaStringEncoder::new();
-
- for i in 1..=127 {
- let origin_string = create_string(i);
-
- let meta_string = encoder.encode(&origin_string).unwrap();
- assert_ne!(meta_string.encoding, Encoding::Utf8);
- assert_eq!(meta_string.original, origin_string);
-
- let decoder = MetaStringDecoder::new();
- let new_string = decoder
- .decode(&meta_string.bytes, meta_string.encoding)
- .unwrap();
- assert_eq!(new_string, origin_string);
+ let special_chars = vec![('.', '_'), ('.', '$'), ('_', '$')];
+ for (special_char1, special_char2) in special_chars {
+ let encoder = MetaStringEncoder::new(special_char1, special_char2);
+ for i in 1..=127 {
+ let origin_string = create_string(i, special_char1, special_char2);
+ let meta_string = encoder.encode(&origin_string).unwrap();
+ assert_ne!(meta_string.encoding, Encoding::Utf8);
+ assert_eq!(meta_string.original, origin_string);
+ let decoder = MetaStringDecoder::new(special_char1, special_char2);
+ let new_string = decoder
+ .decode(&meta_string.bytes, meta_string.encoding)
+ .unwrap();
+ assert_eq!(new_string, origin_string);
+ }
}
}
#[test]
fn test_encode_empty_string() {
- let encoder = MetaStringEncoder::new();
- let decoder = MetaStringDecoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
+ let decoder = &TYPE_NAME_DECODER;
for encoding in [
Encoding::LowerSpecial,
Encoding::LowerUpperDigitSpecial,
@@ -132,7 +139,7 @@ fn test_encode_empty_string() {
#[test]
fn test_encode_characters_outside_of_lower_special() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let test_string = "abcdefABCDEF1234!@#";
let meta_string = encoder.encode(test_string).unwrap();
assert_eq!(meta_string.encoding, Encoding::Utf8);
@@ -140,8 +147,8 @@ fn test_encode_characters_outside_of_lower_special() {
#[test]
fn test_all_to_upper_special_encoding() {
- let encoder = MetaStringEncoder::new();
- let decoder = MetaStringDecoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
+ let decoder = &TYPE_NAME_DECODER;
let test_string = "ABC_DEF";
let meta_string = encoder.encode(test_string).unwrap();
assert_eq!(meta_string.encoding, Encoding::LowerUpperDigitSpecial);
@@ -153,8 +160,8 @@ fn test_all_to_upper_special_encoding() {
#[test]
fn test_first_to_lower_special_encoding() {
- let encoder = MetaStringEncoder::new();
- let decoder = MetaStringDecoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
+ let decoder = &TYPE_NAME_DECODER;
let test_string = "Aabcdef";
let meta_string = encoder.encode(test_string).unwrap();
assert_eq!(meta_string.encoding, Encoding::FirstToLowerSpecial);
@@ -166,11 +173,11 @@ fn test_first_to_lower_special_encoding() {
#[test]
fn test_utf8_encoding() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &NAMESPACE_ENCODER;
let test_string = "你好，世界";
let meta_string = encoder.encode(test_string).unwrap();
assert_eq!(meta_string.encoding, Encoding::Utf8);
- let decoder = MetaStringDecoder::new();
+ let decoder = &TYPE_NAME_DECODER;
let decoded_string = decoder
.decode(&meta_string.bytes, meta_string.encoding)
.unwrap();
@@ -178,7 +185,7 @@ fn test_utf8_encoding() {
let test_string = "aA$";
let meta_string = encoder.encode(test_string).unwrap();
assert_eq!(meta_string.encoding, Encoding::Utf8);
- let decoder = MetaStringDecoder::new();
+ let decoder = &TYPE_NAME_DECODER;
let decoded_string = decoder
.decode(&meta_string.bytes, meta_string.encoding)
.unwrap();
@@ -187,7 +194,7 @@ fn test_utf8_encoding() {
#[test]
fn test_strip_last_char() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let test_string = "abc";
let encoded_meta_string = encoder.encode(test_string).unwrap();
assert!(!encoded_meta_string.strip_last_char);
@@ -199,8 +206,8 @@ fn test_strip_last_char() {
#[test]
fn test_empty_string() {
- let encoder = MetaStringEncoder::new();
- let decoder = MetaStringDecoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
+ let decoder = &TYPE_NAME_DECODER;
let meta_string = encoder.encode("").unwrap();
assert!(meta_string.bytes.is_empty());
let decoded = decoder
@@ -211,7 +218,7 @@ fn test_empty_string() {
#[test]
fn test_ascii_encoding() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let test_string = "asciiOnly";
let encoded_meta_string = encoder.encode(test_string).unwrap();
assert_ne!(encoded_meta_string.encoding, Encoding::Utf8);
@@ -220,7 +227,7 @@ fn test_ascii_encoding() {
#[test]
fn test_non_ascii_encoding() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let test_string = "こんにちは";
let encoded_meta_string = encoder.encode(test_string).unwrap();
assert_eq!(encoded_meta_string.encoding, Encoding::Utf8);
@@ -228,7 +235,7 @@ fn test_non_ascii_encoding() {
#[test]
fn test_non_ascii_encoding_and_non_utf8() {
- let encoder = MetaStringEncoder::new();
+ let encoder = &TYPE_NAME_ENCODER;
let non_ascii_string = "こんにちは";
match encoder.encode_with_encoding(non_ascii_string, Encoding::LowerSpecial) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org