You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fory.apache.org by "urlyy (via GitHub)" <gi...@apache.org> on 2025/09/07 20:14:21 UTC
[PR] feat(Rust): Try to test with java [fory]
urlyy opened a new pull request, #2585:
URL: https://github.com/apache/fory/pull/2585
## What does this PR do?
1. Add files to test with java.
2. Update buffer.var_intxx() && buffer.var_uintxx() and test with java.
3. Update some write logic to sync with java.
## Related issues
#2539
## Does this PR introduce any user-facing change?
- [x] Does this PR introduce any public API change?
```rust
# now can
let reader = Reader::new(bytes.as_slice());
let fory = Fory::default().mode(Compatible).xlang(true);
let mut context = ReadContext::new(&fory, reader);
let a = fory.deserialize_with_context(&mut context);
let b = fory.deserialize_with_context(&mut context);
let c = fory.deserialize_with_context(&mut context);
```
- [x] Does this PR introduce any binary protocol compatibility change?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2341152925
##########
rust/fory-core/src/buffer.rs:
##########
@@ -88,70 +88,209 @@ impl Writer {
}
pub fn var_int32(&mut self, value: i32) {
- if value >> 7 == 0 {
+ let zigzag = ((value as i64) << 1) ^ ((value as i64) >> 31);
+ self.internal_var_uint32(zigzag as u32)
+ }
+
+ pub fn var_uint32(&mut self, value: u32) {
+ self.internal_var_uint32(value)
+ }
+
+ fn internal_var_uint32(&mut self, value: u32) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- self.u16(((u2 << 8) | u1) as u16);
- self.u8((value >> 14) as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
+ } else if value < 0x4000 {
+ // 2 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ // 3 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ // 4 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
- self.u8((value >> 28) as u8);
+ // 5 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
}
}
- pub fn var_uint32(&mut self, value: u32) {
- if value >> 7 == 0 {
+ pub fn var_int64(&mut self, value: i64) {
+ let zigzag = ((value << 1) ^ (value >> 63)) as u64;
+ self.internal_var_uint64(zigzag)
+ }
+
+ pub fn var_uint64(&mut self, value: u64) {
+ self.internal_var_uint64(value)
+ }
+
+ fn internal_var_uint64(&mut self, value: u64) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = value >> 14;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32((u4 << 24) | (u3 << 16) | (u2 << 8) | u1);
+ } else if value < 0x4000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ } else if value < 0x800000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
+ } else if value < 0x40000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (value >> 35) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ } else if value < 0x2000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (value >> 42) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ self.u8(u7);
+ } else if value < 0x100000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (value >> 49) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- let u5 = value >> 28;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- self.u8(u4 as u8);
- self.u8(u5 as u8);
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (((value >> 49) as u8) & 0x7F) | 0x80;
+ let u9 = (value >> 56) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
+ self.u8(u9);
}
}
- pub fn bytes(&mut self, v: &[u8]) {
+ pub fn var_uint36_small(&mut self, value: u64) {
+ assert!(value < (1u64 << 36), "value too large for 36-bit varint");
+ if value < 0x80 {
+ self.u8(value as u8);
+ } else if value < 0x4000 {
+ let b0 = ((value & 0x7F) as u8) | 0x80;
+ let b1 = (value >> 7) as u8;
+ let combined = ((b1 as u16) << 8) | (b0 as u16);
+ self.u16(combined);
+ } else if value < 0x200000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = value >> 14;
+ let combined = b0 | (b1 << 8) | (b2 << 16);
+ self.u32(combined as u32);
+ } else if value < 0x10000000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = value >> 21;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+ self.u32(combined as u32);
+ } else {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = ((value >> 21) & 0x7F) | 0x80;
+ let b4 = value >> 28;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) | (b4 << 32);
+ self.u64(combined);
+ }
+ }
+
+ pub fn latin1_string(&mut self, s: &str) -> usize {
+ for c in s.chars() {
+ let b = c as u32;
+ assert!(b <= 0xFF, "Non-Latin1 character found");
+ self.u8(b as u8);
+ }
+ s.chars().count()
Review Comment:
THis will parse anditerate the string twice, `s.chars()` return an iterator. we'd bettter use a counter to count the length
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2341157487
##########
rust/fory-core/src/buffer.rs:
##########
@@ -88,70 +88,209 @@ impl Writer {
}
pub fn var_int32(&mut self, value: i32) {
- if value >> 7 == 0 {
+ let zigzag = ((value as i64) << 1) ^ ((value as i64) >> 31);
+ self.internal_var_uint32(zigzag as u32)
+ }
+
+ pub fn var_uint32(&mut self, value: u32) {
+ self.internal_var_uint32(value)
+ }
+
+ fn internal_var_uint32(&mut self, value: u32) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- self.u16(((u2 << 8) | u1) as u16);
- self.u8((value >> 14) as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
+ } else if value < 0x4000 {
+ // 2 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ // 3 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ // 4 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
- self.u8((value >> 28) as u8);
+ // 5 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
}
}
- pub fn var_uint32(&mut self, value: u32) {
- if value >> 7 == 0 {
+ pub fn var_int64(&mut self, value: i64) {
+ let zigzag = ((value << 1) ^ (value >> 63)) as u64;
+ self.internal_var_uint64(zigzag)
+ }
+
+ pub fn var_uint64(&mut self, value: u64) {
+ self.internal_var_uint64(value)
+ }
+
+ fn internal_var_uint64(&mut self, value: u64) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = value >> 14;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32((u4 << 24) | (u3 << 16) | (u2 << 8) | u1);
+ } else if value < 0x4000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ } else if value < 0x800000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
+ } else if value < 0x40000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (value >> 35) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ } else if value < 0x2000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (value >> 42) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ self.u8(u7);
+ } else if value < 0x100000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (value >> 49) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- let u5 = value >> 28;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- self.u8(u4 as u8);
- self.u8(u5 as u8);
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (((value >> 49) as u8) & 0x7F) | 0x80;
+ let u9 = (value >> 56) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
+ self.u8(u9);
}
}
- pub fn bytes(&mut self, v: &[u8]) {
+ pub fn var_uint36_small(&mut self, value: u64) {
+ assert!(value < (1u64 << 36), "value too large for 36-bit varint");
+ if value < 0x80 {
+ self.u8(value as u8);
+ } else if value < 0x4000 {
+ let b0 = ((value & 0x7F) as u8) | 0x80;
+ let b1 = (value >> 7) as u8;
+ let combined = ((b1 as u16) << 8) | (b0 as u16);
+ self.u16(combined);
+ } else if value < 0x200000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = value >> 14;
+ let combined = b0 | (b1 << 8) | (b2 << 16);
+ self.u32(combined as u32);
+ } else if value < 0x10000000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = value >> 21;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+ self.u32(combined as u32);
+ } else {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = ((value >> 21) & 0x7F) | 0x80;
+ let b4 = value >> 28;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) | (b4 << 32);
+ self.u64(combined);
+ }
+ }
+
+ pub fn latin1_string(&mut self, s: &str) -> usize {
+ for c in s.chars() {
+ let b = c as u32;
+ assert!(b <= 0xFF, "Non-Latin1 character found");
+ self.u8(b as u8);
+ }
+ s.chars().count()
+ }
+
+ pub fn utf8_string(&mut self, s: &str) -> usize {
+ let bytes = s.as_bytes();
+ for &b in bytes {
+ self.u8(b);
+ }
+ bytes.len()
+ }
+
+ pub fn utf16_string(&mut self, s: &str) -> usize {
+ let units: Vec<u16> = s.encode_utf16().collect();
Review Comment:
the collection will introduce extra memroy allocation, we'd better keep using iterator and use a coutenr teh count written bytes length
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2337326452
##########
java/fory-core/src/test/java/org/apache/fory/WIPCrossLanguageTest.java:
##########
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.serializer.StringSerializer;
+import org.apache.fory.test.TestUtils;
+import org.apache.fory.util.MurmurHash3;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/** Tests in this class need fory python/rust installed. */
+@Test
+public class WIPCrossLanguageTest extends ForyTestBase {
Review Comment:
Rename it to `RustXlangTest`, and skip it by default. We don't want every contributor isntall rust for developing java.
You can control whether enable it by a env veraible. And this test should only be run in rust ci. You can update rust ci by install fory java, and execute command `cd java/fory-core && mvn test -Dtest=org.apache.fory.RustXlangTest`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2338888026
##########
java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java:
##########
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.serializer.StringSerializer;
+import org.apache.fory.test.TestUtils;
+import org.apache.fory.util.MurmurHash3;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/** Tests in this class need fory python/rust installed. */
+@Test
+public class RustXlangTest extends ForyTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(RustXlangTest.class);
+ private static final String PYTHON_EXECUTABLE = "python";
+ private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language";
+
+ private static final String RUST_EXECUTABLE = "cargo";
+ private static final String RUST_MODULE = "test_cross_language";
+
+ private static final List<String> rustBaseCommand =
+ Arrays.asList(
+ RUST_EXECUTABLE,
+ "test",
+ "--test",
+ RUST_MODULE,
+ "<RUST_TESTCASE>",
+ "--",
+ // Use this to get output
+ "--nocapture",
+ // Run unittest that ignored in rust
+ "--ignored",
+ // Exact match test name rather than prefix matching.
+ "--exact");
+ private static final List<String> pyBaseCommand =
+ Arrays.asList(PYTHON_EXECUTABLE, "-m", PYTHON_MODULE, "<TESTCASE>", "<DATA_FILE>");
+
+ private static final int RUST_TESTCASE_INDEX = 4;
+
+ @BeforeClass
+ public void isPyforyInstalled() {
Review Comment:
```suggestion
public void isRustJavaCIEnabled() {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2340552177
##########
ci/tasks/rust.py:
##########
@@ -76,4 +78,12 @@ def run():
)
for cmd in cmds:
common.exec_cmd(cmd)
- logging.info("Executing fory rust tests succeeds")
+
+ logging.info("Executing Rust <-> Java11 cross-language tests")
+ os.environ["FORY_RUST_JAVA_CI"] = "1"
+ java_dir = os.path.join("..", "java")
+ subprocess.check_call(["mvn", "clean", "install", "-DskipTests"], cwd=java_dir)
+ subprocess.check_call(
+ ["mvn", "test", "-Dtest=org.apache.fory.RustXlangTest"],
+ cwd=java_dir,
Review Comment:
```suggestion
cwd=os.path.join(java, "fory-core"),
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2338890165
##########
java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java:
##########
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.serializer.StringSerializer;
+import org.apache.fory.test.TestUtils;
+import org.apache.fory.util.MurmurHash3;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/** Tests in this class need fory python/rust installed. */
+@Test
+public class RustXlangTest extends ForyTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(RustXlangTest.class);
+ private static final String PYTHON_EXECUTABLE = "python";
+ private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language";
+
+ private static final String RUST_EXECUTABLE = "cargo";
+ private static final String RUST_MODULE = "test_cross_language";
+
+ private static final List<String> rustBaseCommand =
+ Arrays.asList(
+ RUST_EXECUTABLE,
+ "test",
+ "--test",
+ RUST_MODULE,
+ "<RUST_TESTCASE>",
+ "--",
+ // Use this to get output
+ "--nocapture",
+ // Run unittest that ignored in rust
+ "--ignored",
+ // Exact match test name rather than prefix matching.
+ "--exact");
+ private static final List<String> pyBaseCommand =
+ Arrays.asList(PYTHON_EXECUTABLE, "-m", PYTHON_MODULE, "<TESTCASE>", "<DATA_FILE>");
+
+ private static final int RUST_TESTCASE_INDEX = 4;
+
+ @BeforeClass
+ public void isPyforyInstalled() {
+ // TestUtils.verifyPyforyInstalled();
+ }
+
+ @Test(enabled = false)
Review Comment:
remove `enabled` flag, use `isRustJavaCIEnabled` instead.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2337318421
##########
java/fory-core/src/main/java/org/apache/fory/resolver/XtypeResolver.java:
##########
@@ -563,6 +563,7 @@ public void writeClassInfo(MemoryBuffer buffer, ClassInfo classInfo) {
case Types.NAMED_STRUCT:
case Types.NAMED_COMPATIBLE_STRUCT:
case Types.NAMED_EXT:
+ case Types.STRUCT:
Review Comment:
Please merge main branch, and revert this line
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2338917131
##########
rust/fory-core/src/serializer/string.rs:
##########
@@ -23,19 +24,104 @@ use crate::serializer::Serializer;
use crate::types::{ForyGeneralList, TypeId};
use std::mem;
+enum StrEncoding {
+ Latin1 = 0,
+ Utf16 = 1,
+ Utf8 = 2,
+}
+
+fn best_coder(s: &str) -> StrEncoding {
+ let chars: Vec<char> = s.chars().collect();
+ let num_chars = chars.len();
+ if num_chars == 0 {
+ return StrEncoding::Latin1;
+ }
+
+ let sample_num = num_chars.min(64);
+ let vectorized_len = sample_num / 4;
+ let vectorized_chars = vectorized_len * 4;
+
+ let mut ascii_count = 0;
+ let mut latin1_count = 0;
+
+ for i in 0..vectorized_len {
+ let base = i * 4;
+ for j in 0..4 {
+ let c = chars[base + j] as u32;
+ if c <= 0x7F {
+ ascii_count += 1;
+ latin1_count += 1;
+ } else if c <= 0xFF {
+ latin1_count += 1;
+ }
+ }
+ }
+
+ for &c in chars.iter().take(sample_num).skip(vectorized_chars) {
+ let c = c as u32;
+ if c <= 0x7F {
+ ascii_count += 1;
+ latin1_count += 1;
+ } else if c <= 0xFF {
+ latin1_count += 1;
+ }
+ }
+
+ if latin1_count == num_chars || latin1_count == sample_num {
+ StrEncoding::Latin1
+ } else if (ascii_count as f64) >= sample_num as f64 * 0.5 {
+ StrEncoding::Utf8
+ } else {
+ StrEncoding::Utf16
+ }
+}
+
impl Serializer for String {
fn reserved_space() -> usize {
mem::size_of::<i32>()
}
fn write(&self, context: &mut WriteContext) {
- context.writer.var_int32(self.len() as i32);
- context.writer.bytes(self.as_bytes());
+ let encoding = best_coder(self);
Review Comment:
For write, we could keep using utf8 instead, this will minimize the string encoding cost, and other languages support decode utf8
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2341139697
##########
rust/fory-core/src/buffer.rs:
##########
@@ -88,70 +88,209 @@ impl Writer {
}
pub fn var_int32(&mut self, value: i32) {
- if value >> 7 == 0 {
+ let zigzag = ((value as i64) << 1) ^ ((value as i64) >> 31);
+ self.internal_var_uint32(zigzag as u32)
+ }
+
+ pub fn var_uint32(&mut self, value: u32) {
+ self.internal_var_uint32(value)
+ }
+
+ fn internal_var_uint32(&mut self, value: u32) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- self.u16(((u2 << 8) | u1) as u16);
- self.u8((value >> 14) as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
+ } else if value < 0x4000 {
+ // 2 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ // 3 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ // 4 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
- self.u8((value >> 28) as u8);
+ // 5 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
}
}
- pub fn var_uint32(&mut self, value: u32) {
- if value >> 7 == 0 {
+ pub fn var_int64(&mut self, value: i64) {
+ let zigzag = ((value << 1) ^ (value >> 63)) as u64;
+ self.internal_var_uint64(zigzag)
+ }
+
+ pub fn var_uint64(&mut self, value: u64) {
+ self.internal_var_uint64(value)
+ }
+
+ fn internal_var_uint64(&mut self, value: u64) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = value >> 14;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32((u4 << 24) | (u3 << 16) | (u2 << 8) | u1);
+ } else if value < 0x4000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ } else if value < 0x800000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
+ } else if value < 0x40000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (value >> 35) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ } else if value < 0x2000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (value >> 42) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ self.u8(u7);
+ } else if value < 0x100000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (value >> 49) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- let u5 = value >> 28;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- self.u8(u4 as u8);
- self.u8(u5 as u8);
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (((value >> 49) as u8) & 0x7F) | 0x80;
+ let u9 = (value >> 56) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
+ self.u8(u9);
}
}
- pub fn bytes(&mut self, v: &[u8]) {
+ pub fn var_uint36_small(&mut self, value: u64) {
+ assert!(value < (1u64 << 36), "value too large for 36-bit varint");
+ if value < 0x80 {
+ self.u8(value as u8);
+ } else if value < 0x4000 {
+ let b0 = ((value & 0x7F) as u8) | 0x80;
+ let b1 = (value >> 7) as u8;
+ let combined = ((b1 as u16) << 8) | (b0 as u16);
+ self.u16(combined);
+ } else if value < 0x200000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = value >> 14;
+ let combined = b0 | (b1 << 8) | (b2 << 16);
+ self.u32(combined as u32);
+ } else if value < 0x10000000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = value >> 21;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+ self.u32(combined as u32);
+ } else {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = ((value >> 21) & 0x7F) | 0x80;
+ let b4 = value >> 28;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) | (b4 << 32);
+ self.u64(combined);
+ }
+ }
+
+ pub fn latin1_string(&mut self, s: &str) -> usize {
Review Comment:
This is prettly costly, and it looks strange that the buffer has a `latin1_string` method.
We should add a convert_to_latin1 method in string serializer and write the converted string to buffer. And we can get length of encoded latin1 binary.
To reduce memory allocation, we can also cache a buffer in string serializer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2338886664
##########
java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java:
##########
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.serializer.StringSerializer;
+import org.apache.fory.test.TestUtils;
+import org.apache.fory.util.MurmurHash3;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/** Tests in this class need fory python/rust installed. */
+@Test
+public class RustXlangTest extends ForyTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(RustXlangTest.class);
+ private static final String PYTHON_EXECUTABLE = "python";
+ private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language";
+
+ private static final String RUST_EXECUTABLE = "cargo";
+ private static final String RUST_MODULE = "test_cross_language";
+
+ private static final List<String> rustBaseCommand =
+ Arrays.asList(
+ RUST_EXECUTABLE,
+ "test",
+ "--test",
+ RUST_MODULE,
+ "<RUST_TESTCASE>",
+ "--",
+ // Use this to get output
+ "--nocapture",
+ // Run unittest that ignored in rust
+ "--ignored",
+ // Exact match test name rather than prefix matching.
+ "--exact");
+ private static final List<String> pyBaseCommand =
+ Arrays.asList(PYTHON_EXECUTABLE, "-m", PYTHON_MODULE, "<TESTCASE>", "<DATA_FILE>");
+
+ private static final int RUST_TESTCASE_INDEX = 4;
+
+ @BeforeClass
+ public void isPyforyInstalled() {
+ // TestUtils.verifyPyforyInstalled();
Review Comment:
We can configure a env variable such as `FORY_RUST_JAVA_CI`, and in `isRustJavaCIEnabled` method, we check the env variable and skip this whole `RustXlangTest` here if env variable not set to 1
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2340589632
##########
ci/tasks/rust.py:
##########
@@ -76,4 +78,12 @@ def run():
)
for cmd in cmds:
common.exec_cmd(cmd)
- logging.info("Executing fory rust tests succeeds")
+
+ logging.info("Executing Rust <-> Java11 cross-language tests")
+ os.environ["FORY_RUST_JAVA_CI"] = "1"
+ java_dir = os.path.join("..", "java")
+ subprocess.check_call(["mvn", "clean", "install", "-DskipTests"], cwd=java_dir)
+ subprocess.check_call(
+ ["mvn", "test", "-Dtest=org.apache.fory.RustXlangTest"],
+ cwd=os.path.join(java, "fory-core"),
Review Comment:
```suggestion
cwd=os.path.join(java_dir, "fory-core"),
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on PR #2585:
URL: https://github.com/apache/fory/pull/2585#issuecomment-3280714800
string serialization in rust needs to support latin1 encoding. You can take following code as an example for simd latin1 check:
```rust
/// Checks if a UTF-8 string can be losslessly encoded as Latin-1 using SIMD if available.
///
/// This is true if all characters in the string have a Unicode codepoint <= 255.
/// This translates to a byte-level check: the string must not contain any byte >= 0xC4.
pub fn can_be_latin1(s: &str) -> bool {
let bytes = s.as_bytes();
// Runtime feature detection to select the best implementation.
// The functions are guarded by `#[target_feature]`, so the compiler
// generates optimized code for each case. The function pointers are resolved
// at runtime on the first call.
#[cfg(target_arch = "x86_64")]
{
if is_x86_feature_detected!("avx2") {
return unsafe { can_be_latin1_avx2(bytes) };
}
if is_x86_feature_detected!("sse2") {
return unsafe { can_be_latin1_sse2(bytes) };
}
}
// Fallback for non-x86_64 architectures or if no SIMD features are available.
can_be_latin1_scalar(bytes)
}
/// Scalar fallback implementation. Checks byte by byte.
#[inline]
fn can_be_latin1_scalar(bytes: &[u8]) -> bool {
// A simple iterator-based check is clean and often optimized well by the compiler.
!bytes.iter().any(|&b| b >= 0xC4)
}
/// Implementation using SSE2 intrinsics (16-byte vectors).
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "sse2")]
unsafe fn can_be_latin1_sse2(bytes: &[u8]) -> bool {
const CHUNK_SIZE: usize = 16;
let limit = _mm_set1_epi8(0xC3 as i8); // We check for values > 0xC3
let mut i = 0;
while i + CHUNK_SIZE <= bytes.len() {
// Load 16 bytes of data. `loadu` handles unaligned memory.
let chunk = _mm_loadu_si128(bytes.as_ptr().add(i) as *const _);
// This is a common trick for unsigned comparison with signed-only intrinsics.
// We want to check `byte >= 0xC4`. This is equivalent to `byte > 0xC3`.
// The `_mm_cmpgt_epi8` instruction performs a signed comparison (a > b).
// By adding -128 (or XORing with 0x80) to both operands, we can map the
// unsigned range [0, 255] to the signed range [-128, 127] and perform a
// valid comparison.
// `byte > 0xC3` becomes `(byte - 128) > (0xC3 - 128)`.
// `0xC3 - 128 = 195 - 128 = 67`. The `limit` vector holds `0xC3` because
// we're comparing `chunk > limit`, and `_mm_cmpgt_epi8` works on signed i8.
// The values from 0xC4 to 0xFF will correctly be "greater than" 0xC3.
let comparison = _mm_cmpgt_epi8(chunk, limit);
// Create a bitmask from the most significant bit of each byte in the result.
// If any byte in `chunk` was > 0xC3, the corresponding byte in `comparison`
// will be all 1s (0xFF), and its MSB will be 1.
// `movemask` will be non-zero if any invalid byte was found.
if _mm_movemask_epi8(comparison) != 0 {
return false;
}
i += CHUNK_SIZE;
}
// Handle the remainder
can_be_latin1_scalar(&bytes[i..])
}
/// Implementation using AVX2 intrinsics (32-byte vectors).
#[cfg(target_arch = "x86_64")]
#[target_feature(enable = "avx2")]
unsafe fn can_be_latin1_avx2(bytes: &[u8]) -> bool {
const CHUNK_SIZE: usize = 32;
// We want to check for bytes >= 0xC4, which is equivalent to > 0xC3.
// The vector is filled with 0xC3.
let limit = _mm256_set1_epi8(0xC3 as i8);
let mut i = 0;
while i + CHUNK_SIZE <= bytes.len() {
// Load 32 bytes of data.
let chunk = _mm256_loadu_si256(bytes.as_ptr().add(i) as *const _);
// Perform a signed "greater than" comparison.
let comparison = _mm256_cmpgt_epi8(chunk, limit);
// Create a 32-bit mask. If any byte was > 0xC3, the mask will be non-zero.
if _mm256_movemask_epi8(comparison) != 0 {
return false;
}
i += CHUNK_SIZE;
}
// Handle the remainder using the SSE2 or scalar implementation
// to avoid duplicating the tail-handling logic.
can_be_latin1_sse2(&bytes[i..])
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2341139697
##########
rust/fory-core/src/buffer.rs:
##########
@@ -88,70 +88,209 @@ impl Writer {
}
pub fn var_int32(&mut self, value: i32) {
- if value >> 7 == 0 {
+ let zigzag = ((value as i64) << 1) ^ ((value as i64) >> 31);
+ self.internal_var_uint32(zigzag as u32)
+ }
+
+ pub fn var_uint32(&mut self, value: u32) {
+ self.internal_var_uint32(value)
+ }
+
+ fn internal_var_uint32(&mut self, value: u32) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- self.u16(((u2 << 8) | u1) as u16);
- self.u8((value >> 14) as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
+ } else if value < 0x4000 {
+ // 2 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ // 3 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ // 4 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- self.u32(((u4 << 24) | (u3 << 16) | (u2 << 8) | u1) as u32);
- self.u8((value >> 28) as u8);
+ // 5 bytes
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
}
}
- pub fn var_uint32(&mut self, value: u32) {
- if value >> 7 == 0 {
+ pub fn var_int64(&mut self, value: i64) {
+ let zigzag = ((value << 1) ^ (value >> 63)) as u64;
+ self.internal_var_uint64(zigzag)
+ }
+
+ pub fn var_uint64(&mut self, value: u64) {
+ self.internal_var_uint64(value)
+ }
+
+ fn internal_var_uint64(&mut self, value: u64) {
+ if value < 0x80 {
self.u8(value as u8);
- } else if value >> 14 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = value >> 7;
- self.u16(((u2 << 8) | u1) as u16);
- } else if value >> 21 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = value >> 14;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- } else if value >> 28 == 0 {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = value >> 21;
- self.u32((u4 << 24) | (u3 << 16) | (u2 << 8) | u1);
+ } else if value < 0x4000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (value >> 7) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ } else if value < 0x200000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (value >> 14) as u8;
+ self.u16(((u2 as u16) << 8) | u1 as u16);
+ self.u8(u3);
+ } else if value < 0x10000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (value >> 21) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ } else if value < 0x800000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (value >> 28) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u8(u5);
+ } else if value < 0x40000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (value >> 35) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ } else if value < 0x2000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (value >> 42) as u8;
+ self.u32(((u4 as u32) << 24) | ((u3 as u32) << 16) | ((u2 as u32) << 8) | u1 as u32);
+ self.u16(((u6 as u16) << 8) | u5 as u16);
+ self.u8(u7);
+ } else if value < 0x100000000000000 {
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (value >> 49) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
} else {
- let u1 = (value & 0x7F) | 0x80;
- let u2 = ((value >> 7) & 0x7F) | 0x80;
- let u3 = ((value >> 14) & 0x7F) | 0x80;
- let u4 = ((value >> 21) & 0x7F) | 0x80;
- let u5 = value >> 28;
- self.u8(u1 as u8);
- self.u8(u2 as u8);
- self.u8(u3 as u8);
- self.u8(u4 as u8);
- self.u8(u5 as u8);
+ let u1 = ((value as u8) & 0x7F) | 0x80;
+ let u2 = (((value >> 7) as u8) & 0x7F) | 0x80;
+ let u3 = (((value >> 14) as u8) & 0x7F) | 0x80;
+ let u4 = (((value >> 21) as u8) & 0x7F) | 0x80;
+ let u5 = (((value >> 28) as u8) & 0x7F) | 0x80;
+ let u6 = (((value >> 35) as u8) & 0x7F) | 0x80;
+ let u7 = (((value >> 42) as u8) & 0x7F) | 0x80;
+ let u8 = (((value >> 49) as u8) & 0x7F) | 0x80;
+ let u9 = (value >> 56) as u8;
+ self.u64(
+ (u8 as u64) << 56
+ | (u7 as u64) << 48
+ | (u6 as u64) << 40
+ | (u5 as u64) << 32
+ | (u4 as u64) << 24
+ | (u3 as u64) << 16
+ | (u2 as u64) << 8
+ | (u1 as u64),
+ );
+ self.u8(u9);
}
}
- pub fn bytes(&mut self, v: &[u8]) {
+ pub fn var_uint36_small(&mut self, value: u64) {
+ assert!(value < (1u64 << 36), "value too large for 36-bit varint");
+ if value < 0x80 {
+ self.u8(value as u8);
+ } else if value < 0x4000 {
+ let b0 = ((value & 0x7F) as u8) | 0x80;
+ let b1 = (value >> 7) as u8;
+ let combined = ((b1 as u16) << 8) | (b0 as u16);
+ self.u16(combined);
+ } else if value < 0x200000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = value >> 14;
+ let combined = b0 | (b1 << 8) | (b2 << 16);
+ self.u32(combined as u32);
+ } else if value < 0x10000000 {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = value >> 21;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24);
+ self.u32(combined as u32);
+ } else {
+ let b0 = (value & 0x7F) | 0x80;
+ let b1 = ((value >> 7) & 0x7F) | 0x80;
+ let b2 = ((value >> 14) & 0x7F) | 0x80;
+ let b3 = ((value >> 21) & 0x7F) | 0x80;
+ let b4 = value >> 28;
+ let combined = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) | (b4 << 32);
+ self.u64(combined);
+ }
+ }
+
+ pub fn latin1_string(&mut self, s: &str) -> usize {
Review Comment:
This is prettly costly, and it looks strange that the buffer has a `latin1_string` method.
We should add a convert_to_latin1 method in string serializer and write the converted string to buffer. And we can get length of encoded latin1 binary.
To reduce memory allocation, we can also cache a buffer in string serializer.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2341212553
##########
rust/fory-core/src/serializer/string.rs:
##########
@@ -15,27 +15,60 @@
// specific language governing permissions and limitations
// under the License.
+use crate::buffer::Writer;
use crate::error::Error;
use crate::fory::Fory;
+use crate::meta::is_latin;
use crate::resolver::context::ReadContext;
use crate::resolver::context::WriteContext;
use crate::serializer::Serializer;
use crate::types::{ForyGeneralList, TypeId};
use std::mem;
+enum StrEncoding {
+ Latin1 = 0,
+ Utf16 = 1,
+ Utf8 = 2,
+}
+
impl Serializer for String {
fn reserved_space() -> usize {
mem::size_of::<i32>()
}
fn write(&self, context: &mut WriteContext) {
- context.writer.var_int32(self.len() as i32);
- context.writer.bytes(self.as_bytes());
+ let mut buf = Writer::default();
+ let len;
+ let bitor;
+ if is_latin(self.as_str()) {
Review Comment:
Maybe not in this PR, we could implement a get_latin1_length, which return -1 if not latin1. And then write length and encoding first, then we can just write string byte by byte directly into the writer
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang commented on code in PR #2585:
URL: https://github.com/apache/fory/pull/2585#discussion_r2338895139
##########
java/fory-core/src/test/java/org/apache/fory/RustXlangTest.java:
##########
@@ -0,0 +1,507 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fory;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.hash.Hashing;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
+import lombok.Data;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.fory.config.CompatibleMode;
+import org.apache.fory.config.Language;
+import org.apache.fory.logging.Logger;
+import org.apache.fory.logging.LoggerFactory;
+import org.apache.fory.memory.MemoryBuffer;
+import org.apache.fory.memory.MemoryUtils;
+import org.apache.fory.serializer.StringSerializer;
+import org.apache.fory.test.TestUtils;
+import org.apache.fory.util.MurmurHash3;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+/** Tests in this class need fory python/rust installed. */
+@Test
+public class RustXlangTest extends ForyTestBase {
+ private static final Logger LOG = LoggerFactory.getLogger(RustXlangTest.class);
+ private static final String PYTHON_EXECUTABLE = "python";
+ private static final String PYTHON_MODULE = "pyfory.tests.test_cross_language";
+
+ private static final String RUST_EXECUTABLE = "cargo";
+ private static final String RUST_MODULE = "test_cross_language";
+
+ private static final List<String> rustBaseCommand =
+ Arrays.asList(
+ RUST_EXECUTABLE,
+ "test",
+ "--test",
+ RUST_MODULE,
+ "<RUST_TESTCASE>",
+ "--",
+ // Use this to get output
+ "--nocapture",
+ // Run unittest that ignored in rust
+ "--ignored",
+ // Exact match test name rather than prefix matching.
+ "--exact");
+ private static final List<String> pyBaseCommand =
+ Arrays.asList(PYTHON_EXECUTABLE, "-m", PYTHON_MODULE, "<TESTCASE>", "<DATA_FILE>");
+
+ private static final int RUST_TESTCASE_INDEX = 4;
+
+ @BeforeClass
+ public void isPyforyInstalled() {
+ // TestUtils.verifyPyforyInstalled();
Review Comment:
Please also update rust ci in `.github/workflows/ci.yaml` to install fory java too and run test command `mvn test -Dtest=org.apache.fory.RustXlangTest`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org
Re: [PR] feat(Rust): support basic type se/de aligned with java [fory]
Posted by "chaokunyang (via GitHub)" <gi...@apache.org>.
chaokunyang merged PR #2585:
URL: https://github.com/apache/fory/pull/2585
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@fory.apache.org
For additional commands, e-mail: commits-help@fory.apache.org