You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/08/08 18:49:41 UTC
[avro] branch master updated: AVRO-3599: Rust: Make apache-avro-test-helper releasable (#1815)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/master by this push:
new 089e40f9c AVRO-3599: Rust: Make apache-avro-test-helper releasable (#1815)
089e40f9c is described below
commit 089e40f9ca0253962009fd277995a627a65620eb
Author: Martin Grigorov <ma...@users.noreply.github.com>
AuthorDate: Mon Aug 8 21:49:35 2022 +0300
AVRO-3599: Rust: Make apache-avro-test-helper releasable (#1815)
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
---
lang/rust/Cargo.lock | 116 +++----
lang/rust/README.md | 637 +---------------------------------
lang/rust/avro/Cargo.toml | 2 +-
lang/rust/{ => avro}/README.md | 0
lang/rust/avro_test_helper/Cargo.toml | 13 +-
5 files changed, 67 insertions(+), 701 deletions(-)
diff --git a/lang/rust/Cargo.lock b/lang/rust/Cargo.lock
index 2bc3d2e42..41c4c82a7 100644
--- a/lang/rust/Cargo.lock
+++ b/lang/rust/Cargo.lock
@@ -90,7 +90,7 @@ dependencies = [
[[package]]
name = "apache-avro-test-helper"
-version = "0.1.0"
+version = "0.14.0"
dependencies = [
"color-backtrace",
"ctor",
@@ -119,9 +119,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "backtrace"
-version = "0.3.65"
+version = "0.3.66"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
+checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
dependencies = [
"addr2line",
"cc",
@@ -192,15 +192,6 @@ dependencies = [
"pkg-config",
]
-[[package]]
-name = "cast"
-version = "0.2.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c24dab4283a142afa2fdca129b80ad2c6284e073930f964c3a1293c225ee39a"
-dependencies = [
- "rustc_version",
-]
-
[[package]]
name = "cast"
version = "0.3.0"
@@ -279,7 +270,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f"
dependencies = [
"atty",
- "cast 0.3.0",
+ "cast",
"clap",
"criterion-plot",
"csv",
@@ -300,19 +291,19 @@ dependencies = [
[[package]]
name = "criterion-plot"
-version = "0.4.4"
+version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d00996de9f2f7559f7f4dc286073197f83e92256a59ed395f9aac01fe717da57"
+checksum = "2673cc8207403546f45f5fd319a974b1e6983ad1a3ee7e6041650013be041876"
dependencies = [
- "cast 0.2.7",
+ "cast",
"itertools",
]
[[package]]
name = "crossbeam-channel"
-version = "0.5.5"
+version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c02a4d71819009c192cf4872265391563fd6a84c81ff2c0f2a7026ca4c1d85c"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
dependencies = [
"cfg-if",
"crossbeam-utils",
@@ -320,9 +311,9 @@ dependencies = [
[[package]]
name = "crossbeam-deque"
-version = "0.8.1"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
dependencies = [
"cfg-if",
"crossbeam-epoch",
@@ -331,9 +322,9 @@ dependencies = [
[[package]]
name = "crossbeam-epoch"
-version = "0.9.9"
+version = "0.9.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07db9d94cbd326813772c968ccd25999e5f8ae22f4f8d1b11effa37ef6ce281d"
+checksum = "045ebe27666471bb549370b4b0b3e51b07f56325befa4284db65fc89c02511b1"
dependencies = [
"autocfg",
"cfg-if",
@@ -345,9 +336,9 @@ dependencies = [
[[package]]
name = "crossbeam-utils"
-version = "0.8.10"
+version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d82ee10ce34d7bc12c2122495e7593a9c41347ecdd64185af4ecf72cb1a7f83"
+checksum = "51887d4adc7b564537b15adcfb307936f8075dfcd5f00dde9a9f1d29383682bc"
dependencies = [
"cfg-if",
"once_cell",
@@ -355,9 +346,9 @@ dependencies = [
[[package]]
name = "crypto-common"
-version = "0.1.3"
+version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
"typenum",
@@ -431,9 +422,9 @@ dependencies = [
[[package]]
name = "diff"
-version = "0.1.12"
+version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
+checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "digest"
@@ -447,9 +438,9 @@ dependencies = [
[[package]]
name = "either"
-version = "1.6.1"
+version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be"
[[package]]
name = "env_logger"
@@ -468,9 +459,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "generic-array"
-version = "0.14.5"
+version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
dependencies = [
"typenum",
"version_check",
@@ -489,9 +480,9 @@ dependencies = [
[[package]]
name = "gimli"
-version = "0.26.1"
+version = "0.26.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
+checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
[[package]]
name = "half"
@@ -554,9 +545,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
-version = "1.0.2"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754"
[[package]]
name = "jobserver"
@@ -584,9 +575,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
-version = "0.2.126"
+version = "0.2.127"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+checksum = "505e71a4706fa491e9b1b55f51b95d4037d0821ee40131190475f692b35b009b"
[[package]]
name = "libflate"
@@ -703,18 +694,18 @@ dependencies = [
[[package]]
name = "object"
-version = "0.28.4"
+version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424"
+checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
dependencies = [
"memchr",
]
[[package]]
name = "once_cell"
-version = "1.12.0"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"
+checksum = "18a6dbe30758c9f83eb00cbea4ac95966305f5a7772f3f42ebfc7fc7eddbd8e1"
[[package]]
name = "oorandom"
@@ -752,15 +743,15 @@ dependencies = [
[[package]]
name = "plotters-backend"
-version = "0.3.2"
+version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d88417318da0eaf0fdcdb51a0ee6c3bed624333bff8f946733049380be67ac1c"
+checksum = "193228616381fecdc1224c62e96946dfbc73ff4384fba576e052ff8c1bea8142"
[[package]]
name = "plotters-svg"
-version = "0.3.1"
+version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "521fa9638fa597e1dc53e9412a4f9cefb01187ee1f7413076f9e6749e2885ba9"
+checksum = "e0918736323d1baff32ee0eade54984f6f201ad7e97d5cfb5d6ab4a358529615"
dependencies = [
"plotters-backend",
]
@@ -932,26 +923,17 @@ version = "0.1.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
-[[package]]
-name = "rustc_version"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
-dependencies = [
- "semver",
-]
-
[[package]]
name = "rustversion"
-version = "1.0.7"
+version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0a5f7c728f5d284929a1cccb5bc19884422bfe6ef4d6c409da2c41838983fcf"
+checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8"
[[package]]
name = "ryu"
-version = "1.0.10"
+version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09"
[[package]]
name = "same-file"
@@ -974,12 +956,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
-[[package]]
-name = "semver"
-version = "1.0.10"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a41d061efea015927ac527063765e73601444cdc344ba855bc7bd44578b25e1c"
-
[[package]]
name = "serde"
version = "1.0.142"
@@ -1016,7 +992,7 @@ version = "1.0.83"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7"
dependencies = [
- "itoa 1.0.2",
+ "itoa 1.0.3",
"ryu",
"serde",
]
@@ -1147,9 +1123,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
[[package]]
name = "unicode-ident"
-version = "1.0.1"
+version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5bd2fe26506023ed7b5e1e315add59d6f584c621d037f9368fea9cfb988f368c"
+checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf"
[[package]]
name = "unicode-width"
@@ -1287,9 +1263,9 @@ dependencies = [
[[package]]
name = "web-sys"
-version = "0.3.58"
+version = "0.3.59"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fed94beee57daf8dd7d51f2b15dc2bcde92d7a72304cdf662a4371008b71b90"
+checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1"
dependencies = [
"js-sys",
"wasm-bindgen",
diff --git a/lang/rust/README.md b/lang/rust/README.md
index 965dd51ab..199154393 100644
--- a/lang/rust/README.md
+++ b/lang/rust/README.md
@@ -19,640 +19,23 @@
# apache-avro
-[](https://crates.io/crates/apache-avro)
-[](https://github.com/apache/avro/actions/workflows/test-lang-rust-ci.yml)
-[](https://docs.rs/apache-avro)
-[](https://github.com/apache/avro/blob/master/LICENSE.txt)
+Apache Avro Rust SDK
-A library for working with [Apache Avro](https://avro.apache.org/) in Rust language.
+# Subprojects
-Please check our [documentation](https://docs.rs/apache-avro) for examples, tutorials and API reference.
+## Avro
-**[Apache Avro](https://avro.apache.org/)** is a data serialization system which provides rich
-data structures and a compact, fast, binary data format.
+See [avro/README.md](./avro/README.md)
-All data in Avro is schematized, as in the following example:
+## Avro derive
-```
-{
- "type": "record",
- "name": "test",
- "fields": [
- {"name": "a", "type": "long", "default": 42},
- {"name": "b", "type": "string"}
- ]
-}
-```
+See [avro_derive/README.md](./avro_derive/README.md)
-There are basically two ways of handling Avro data in Rust:
+## Avro test helper
-* **as Avro-specialized data types** based on an Avro schema;
-* **as generic Rust serde-compatible types** implementing/deriving `Serialize` and
-`Deserialize`;
+See [avro_test_helper/README.md](./avro_test_helper/README.md)
-**apache-avro** provides a way to read and write both these data representations easily and
-efficiently.
+## WebAssembly demo application
-## Installing the library
+See [wasm-demo/README.md](./wasm-demo/README.md)
-
-Add to your `Cargo.toml`:
-
-```toml
-[dependencies]
-apache-avro = "x.y"
-```
-
-Or in case you want to leverage the **Snappy** codec:
-
-```toml
-[dependencies.apache-avro]
-version = "x.y"
-features = ["snappy"]
-```
-
-Or in case you want to leverage the **Zstandard** codec:
-
-```toml
-[dependencies.apache-avro]
-version = "x.y"
-features = ["zstandard"]
-```
-
-Or in case you want to leverage the **Bzip2** codec:
-
-```toml
-[dependencies.apache-avro]
-version = "x.y"
-features = ["bzip"]
-```
-
-Or in case you want to leverage the **Xz** codec:
-
-```toml
-[dependencies.apache-avro]
-version = "x.y"
-features = ["xz"]
-```
-
-
-## Upgrading to a newer minor version
-
-The library is still in beta, so there might be backward-incompatible changes between minor
-versions. If you have troubles upgrading, check the [version upgrade guide](migration_guide.md).
-
-## Defining a schema
-
-An Avro data cannot exist without an Avro schema. Schemas **must** be used while writing and
-**can** be used while reading and they carry the information regarding the type of data we are
-handling. Avro schemas are used for both schema validation and resolution of Avro data.
-
-Avro schemas are defined in **JSON** format and can just be parsed out of a raw string:
-
-```rust
-use apache_avro::Schema;
-
-let raw_schema = r#"
- {
- "type": "record",
- "name": "test",
- "fields": [
- {"name": "a", "type": "long", "default": 42},
- {"name": "b", "type": "string"}
- ]
- }
-"#;
-
-// if the schema is not valid, this function will return an error
-let schema = Schema::parse_str(raw_schema).unwrap();
-
-// schemas can be printed for debugging
-println!("{:?}", schema);
-```
-
-Additionally, a list of of definitions (which may depend on each other) can be given and all of
-them will be parsed into the corresponding schemas.
-
-```rust
-use apache_avro::Schema;
-
-let raw_schema_1 = r#"{
- "name": "A",
- "type": "record",
- "fields": [
- {"name": "field_one", "type": "float"}
- ]
- }"#;
-
-// This definition depends on the definition of A above
-let raw_schema_2 = r#"{
- "name": "B",
- "type": "record",
- "fields": [
- {"name": "field_one", "type": "A"}
- ]
- }"#;
-
-// if the schemas are not valid, this function will return an error
-let schemas = Schema::parse_list(&[raw_schema_1, raw_schema_2]).unwrap();
-
-// schemas can be printed for debugging
-println!("{:?}", schemas);
-```
-*N.B.* It is important to note that the composition of schema definitions requires schemas with names.
-For this reason, only schemas of type Record, Enum, and Fixed should be input into this function.
-
-The library provides also a programmatic interface to define schemas without encoding them in
-JSON (for advanced use), but we highly recommend the JSON interface. Please read the API
-reference in case you are interested.
-
-For more information about schemas and what kind of information you can encapsulate in them,
-please refer to the appropriate section of the
-[Avro Specification](https://avro.apache.org/docs/current/spec.html#schemas).
-
-## Writing data
-
-Once we have defined a schema, we are ready to serialize data in Avro, validating them against
-the provided schema in the process. As mentioned before, there are two ways of handling Avro
-data in Rust.
-
-**NOTE:** The library also provides a low-level interface for encoding a single datum in Avro
-bytecode without generating markers and headers (for advanced use), but we highly recommend the
-`Writer` interface to be totally Avro-compatible. Please read the API reference in case you are
-interested.
-
-### The avro way
-
-Given that the schema we defined above is that of an Avro *Record*, we are going to use the
-associated type provided by the library to specify the data we want to serialize:
-
-```rust
-use apache_avro::types::Record;
-use apache_avro::Writer;
-#
-// a writer needs a schema and something to write to
-let mut writer = Writer::new(&schema, Vec::new());
-
-// the Record type models our Record schema
-let mut record = Record::new(writer.schema()).unwrap();
-record.put("a", 27i64);
-record.put("b", "foo");
-
-// schema validation happens here
-writer.append(record).unwrap();
-
-// this is how to get back the resulting avro bytecode
-// this performs a flush operation to make sure data has been written, so it can fail
-// you can also call `writer.flush()` yourself without consuming the writer
-let encoded = writer.into_inner().unwrap();
-```
-
-The vast majority of the times, schemas tend to define a record as a top-level container
-encapsulating all the values to convert as fields and providing documentation for them, but in
-case we want to directly define an Avro value, the library offers that capability via the
-`Value` interface.
-
-```rust
-use apache_avro::types::Value;
-
-let mut value = Value::String("foo".to_string());
-```
-
-### The serde way
-
-Given that the schema we defined above is an Avro *Record*, we can directly use a Rust struct
-deriving `Serialize` to model our data:
-
-```rust
-use apache_avro::Writer;
-
-#[derive(Debug, Serialize)]
-struct Test {
- a: i64,
- b: String,
-}
-
-// a writer needs a schema and something to write to
-let mut writer = Writer::new(&schema, Vec::new());
-
-// the structure models our Record schema
-let test = Test {
- a: 27,
- b: "foo".to_owned(),
-};
-
-// schema validation happens here
-writer.append_ser(test).unwrap();
-
-// this is how to get back the resulting avro bytecode
-// this performs a flush operation to make sure data is written, so it can fail
-// you can also call `writer.flush()` yourself without consuming the writer
-let encoded = writer.into_inner();
-```
-
-The vast majority of the times, schemas tend to define a record as a top-level container
-encapsulating all the values to convert as fields and providing documentation for them, but in
-case we want to directly define an Avro value, any type implementing `Serialize` should work.
-
-```rust
-let mut value = "foo".to_string();
-```
-
-### Using codecs to compress data
-
-Avro supports three different compression codecs when encoding data:
-
-* **Null**: leaves data uncompressed;
-* **Deflate**: writes the data block using the deflate algorithm as specified in RFC 1951, and
-typically implemented using the zlib library. Note that this format (unlike the "zlib format" in
-RFC 1950) does not have a checksum.
-* **Snappy**: uses Google's [Snappy](http://google.github.io/snappy/) compression library. Each
-compressed block is followed by the 4-byte, big-endianCRC32 checksum of the uncompressed data in
-the block. You must enable the `snappy` feature to use this codec.
-* **Zstandard**: uses Facebook's [Zstandard](https://facebook.github.io/zstd/) compression library.
-You must enable the `zstandard` feature to use this codec.
-* **Bzip2**: uses [BZip2](https://sourceware.org/bzip2/) compression library.
-You must enable the `bzip` feature to use this codec.
-* **Xz**: uses [xz2](https://github.com/alexcrichton/xz2-rs) compression library.
- You must enable the `xz` feature to use this codec.
-
-
-To specify a codec to use to compress data, just specify it while creating a `Writer`:
-```rust
-use apache_avro::Writer;
-use apache_avro::Codec;
-#
-let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate);
-```
-
-## Reading data
-
-As far as reading Avro encoded data goes, we can just use the schema encoded with the data to
-read them. The library will do it automatically for us, as it already does for the compression
-codec:
-
-```rust
-use apache_avro::Reader;
-#
-// reader creation can fail in case the input to read from is not Avro-compatible or malformed
-let reader = Reader::new(&input[..]).unwrap();
-```
-
-In case, instead, we want to specify a different (but compatible) reader schema from the schema
-the data has been written with, we can just do as the following:
-```rust
-use apache_avro::Schema;
-use apache_avro::Reader;
-#
-
-let reader_raw_schema = r#"
- {
- "type": "record",
- "name": "test",
- "fields": [
- {"name": "a", "type": "long", "default": 42},
- {"name": "b", "type": "string"},
- {"name": "c", "type": "long", "default": 43}
- ]
- }
-"#;
-
-let reader_schema = Schema::parse_str(reader_raw_schema).unwrap();
-
-// reader creation can fail in case the input to read from is not Avro-compatible or malformed
-let reader = Reader::with_schema(&reader_schema, &input[..]).unwrap();
-```
-
-The library will also automatically perform schema resolution while reading the data.
-
-For more information about schema compatibility and resolution, please refer to the
-[Avro Specification](https://avro.apache.org/docs/current/spec.html#schemas).
-
-As usual, there are two ways to handle Avro data in Rust, as you can see below.
-
-**NOTE:** The library also provides a low-level interface for decoding a single datum in Avro
-bytecode without markers and header (for advanced use), but we highly recommend the `Reader`
-interface to leverage all Avro features. Please read the API reference in case you are
-interested.
-
-
-### The avro way
-
-We can just read directly instances of `Value` out of the `Reader` iterator:
-
-```rust
-use apache_avro::Reader;
-#
-let reader = Reader::new(&input[..]).unwrap();
-
-// value is a Result of an Avro Value in case the read operation fails
-for value in reader {
- println!("{:?}", value.unwrap());
-}
-
-```
-
-### The serde way
-
-Alternatively, we can use a Rust type implementing `Deserialize` and representing our schema to
-read the data into:
-
-```rust
-use apache_avro::Reader;
-use apache_avro::from_value;
-
-#[derive(Debug, Deserialize)]
-struct Test {
- a: i64,
- b: String,
-}
-
-let reader = Reader::new(&input[..]).unwrap();
-
-// value is a Result in case the read operation fails
-for value in reader {
- println!("{:?}", from_value::<Test>(&value.unwrap()));
-}
-```
-
-## Putting everything together
-
-The following is an example of how to combine everything showed so far and it is meant to be a
-quick reference of the library interface:
-
-```rust
-use apache_avro::{Codec, Reader, Schema, Writer, from_value, types::Record, Error};
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Deserialize, Serialize)]
-struct Test {
- a: i64,
- b: String,
-}
-
-fn main() -> Result<(), Error> {
- let raw_schema = r#"
- {
- "type": "record",
- "name": "test",
- "fields": [
- {"name": "a", "type": "long", "default": 42},
- {"name": "b", "type": "string"}
- ]
- }
- "#;
-
- let schema = Schema::parse_str(raw_schema)?;
-
- println!("{:?}", schema);
-
- let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate);
-
- let mut record = Record::new(writer.schema()).unwrap();
- record.put("a", 27i64);
- record.put("b", "foo");
-
- writer.append(record)?;
-
- let test = Test {
- a: 27,
- b: "foo".to_owned(),
- };
-
- writer.append_ser(test)?;
-
- let input = writer.into_inner()?;
- let reader = Reader::with_schema(&schema, &input[..])?;
-
- for record in reader {
- println!("{:?}", from_value::<Test>(&record?));
- }
- Ok(())
-}
-```
-
-`apache-avro` also supports the logical types listed in the [Avro specification](https://avro.apache.org/docs/current/spec.html#Logical+Types):
-
-1. `Decimal` using the [`num_bigint`](https://docs.rs/num-bigint/0.2.6/num_bigint) crate
-1. UUID using the [`uuid`](https://docs.rs/uuid/0.8.1/uuid) crate
-1. Date, Time (milli) as `i32` and Time (micro) as `i64`
-1. Timestamp (milli and micro) as `i64`
-1. Duration as a custom type with `months`, `days` and `millis` accessor methods each of which returns an `i32`
-
-Note that the on-disk representation is identical to the underlying primitive/complex type.
-
-#### Read and write logical types
-
-```rust
-use apache_avro::{
- types::Record, types::Value, Codec, Days, Decimal, Duration, Millis, Months, Reader, Schema,
- Writer, Error,
-};
-use num_bigint::ToBigInt;
-
-fn main() -> Result<(), Error> {
- let raw_schema = r#"
- {
- "type": "record",
- "name": "test",
- "fields": [
- {
- "name": "decimal_fixed",
- "type": {
- "type": "fixed",
- "size": 2,
- "name": "decimal"
- },
- "logicalType": "decimal",
- "precision": 4,
- "scale": 2
- },
- {
- "name": "decimal_var",
- "type": "bytes",
- "logicalType": "decimal",
- "precision": 10,
- "scale": 3
- },
- {
- "name": "uuid",
- "type": "string",
- "logicalType": "uuid"
- },
- {
- "name": "date",
- "type": "int",
- "logicalType": "date"
- },
- {
- "name": "time_millis",
- "type": "int",
- "logicalType": "time-millis"
- },
- {
- "name": "time_micros",
- "type": "long",
- "logicalType": "time-micros"
- },
- {
- "name": "timestamp_millis",
- "type": "long",
- "logicalType": "timestamp-millis"
- },
- {
- "name": "timestamp_micros",
- "type": "long",
- "logicalType": "timestamp-micros"
- },
- {
- "name": "duration",
- "type": {
- "type": "fixed",
- "size": 12,
- "name": "duration"
- },
- "logicalType": "duration"
- }
- ]
- }
- "#;
-
- let schema = Schema::parse_str(raw_schema)?;
-
- println!("{:?}", schema);
-
- let mut writer = Writer::with_codec(&schema, Vec::new(), Codec::Deflate);
-
- let mut record = Record::new(writer.schema()).unwrap();
- record.put("decimal_fixed", Decimal::from(9936.to_bigint().unwrap().to_signed_bytes_be()));
- record.put("decimal_var", Decimal::from((-32442.to_bigint().unwrap()).to_signed_bytes_be()));
- record.put("uuid", uuid::Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").unwrap());
- record.put("date", Value::Date(1));
- record.put("time_millis", Value::TimeMillis(2));
- record.put("time_micros", Value::TimeMicros(3));
- record.put("timestamp_millis", Value::TimestampMillis(4));
- record.put("timestamp_micros", Value::TimestampMicros(5));
- record.put("duration", Duration::new(Months::new(6), Days::new(7), Millis::new(8)));
-
- writer.append(record)?;
-
- let input = writer.into_inner()?;
- let reader = Reader::with_schema(&schema, &input[..])?;
-
- for record in reader {
- println!("{:?}", record?);
- }
- Ok(())
-}
-```
-
-### Calculate Avro schema fingerprint
-
-This library supports calculating the following fingerprints:
-
- - SHA-256
- - MD5
- - Rabin
-
-An example of fingerprinting for the supported fingerprints:
-
-```rust
-use apache_avro::rabin::Rabin;
-use apache_avro::{Schema, Error};
-use md5::Md5;
-use sha2::Sha256;
-
-fn main() -> Result<(), Error> {
- let raw_schema = r#"
- {
- "type": "record",
- "name": "test",
- "fields": [
- {"name": "a", "type": "long", "default": 42},
- {"name": "b", "type": "string"}
- ]
- }
- "#;
- let schema = Schema::parse_str(raw_schema)?;
- println!("{}", schema.fingerprint::<Sha256>());
- println!("{}", schema.fingerprint::<Md5>());
- println!("{}", schema.fingerprint::<Rabin>());
- Ok(())
-}
-```
-
-### Ill-formed data
-
-In order to ease decoding, the Binary Encoding specification of Avro data
-requires some fields to have their length encoded alongside the data.
-
-If encoded data passed to a `Reader` has been ill-formed, it can happen that
-the bytes meant to contain the length of data are bogus and could result
-in extravagant memory allocation.
-
-To shield users from ill-formed data, `apache-avro` sets a limit (default: 512MB)
-to any allocation it will perform when decoding data.
-
-If you expect some of your data fields to be larger than this limit, be sure
-to make use of the `max_allocation_bytes` function before reading **any** data
-(we leverage Rust's [`std::sync::Once`](https://doc.rust-lang.org/std/sync/struct.Once.html)
-mechanism to initialize this value, if
-any call to decode is made before a call to `max_allocation_bytes`, the limit
-will be 512MB throughout the lifetime of the program).
-
-
-```rust
-use apache_avro::max_allocation_bytes;
-
-max_allocation_bytes(2 * 1024 * 1024 * 1024); // 2GB
-
-// ... happily decode large data
-
-```
-
-### Check schemas compatibility
-
-This library supports checking for schemas compatibility.
-
-Examples of checking for compatibility:
-
-1. Compatible schemas
-
-Explanation: an int array schema can be read by a long array schema- an int
-(32bit signed integer) fits into a long (64bit signed integer)
-
-```rust
-use apache_avro::{Schema, schema_compatibility::SchemaCompatibility};
-
-let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap();
-let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap();
-assert_eq!(true, SchemaCompatibility::can_read(&writers_schema, &readers_schema));
-```
-
-2. Incompatible schemas (a long array schema cannot be read by an int array schema)
-
-Explanation: a long array schema cannot be read by an int array schema- a
-long (64bit signed integer) does not fit into an int (32bit signed integer)
-
-```rust
-use apache_avro::{Schema, schema_compatibility::SchemaCompatibility};
-
-let writers_schema = Schema::parse_str(r#"{"type": "array", "items":"long"}"#).unwrap();
-let readers_schema = Schema::parse_str(r#"{"type": "array", "items":"int"}"#).unwrap();
-assert_eq!(false, SchemaCompatibility::can_read(&writers_schema, &readers_schema));
-```
-
-## Minimal supported Rust version
-
-1.54.0
-
-## License
-This project is licensed under [Apache License 2.0](https://github.com/apache/avro/blob/master/LICENSE.txt).
-
-## Contributing
-Everyone is encouraged to contribute! You can contribute by forking the GitHub repo and making a pull request or opening an issue.
-All contributions will be licensed under [Apache License 2.0](https://github.com/apache/avro/blob/master/LICENSE.txt).
-
-Please consider adding documentation and tests!
-If you introduce a backward-incompatible change, please consider adding instruction to migrate in the [Migration Guide](migration_guide.md)
-If you modify the crate documentation in `lib.rs`, run `make readme` to sync the README file.
diff --git a/lang/rust/avro/Cargo.toml b/lang/rust/avro/Cargo.toml
index d1d0292a3..f47f8f97b 100644
--- a/lang/rust/avro/Cargo.toml
+++ b/lang/rust/avro/Cargo.toml
@@ -83,7 +83,7 @@ rand = { default-features = false, version = "0.8.5", features = ["default"] }
[dev-dependencies]
anyhow = { default-features = false, version = "1.0.60", features = ["std"] }
-apache-avro-test-helper = { default-features = false, version = "0.1.0", path = "../avro_test_helper" }
+apache-avro-test-helper = { default-features = false, version = "0.14.0", path = "../avro_test_helper" }
criterion = { default-features = false, version = "0.3.6" }
hex-literal = { default-features = false, version = "0.3.4" }
md-5 = { default-features = false, version = "0.10.1" }
diff --git a/lang/rust/README.md b/lang/rust/avro/README.md
similarity index 100%
copy from lang/rust/README.md
copy to lang/rust/avro/README.md
diff --git a/lang/rust/avro_test_helper/Cargo.toml b/lang/rust/avro_test_helper/Cargo.toml
index 0e77a5d55..4f3cdf02a 100644
--- a/lang/rust/avro_test_helper/Cargo.toml
+++ b/lang/rust/avro_test_helper/Cargo.toml
@@ -17,10 +17,17 @@
[package]
name = "apache-avro-test-helper"
-version = "0.1.0"
+version = "0.14.0"
edition = "2018"
-publish = false
-description = "Avro test helper. This crate is not supposed to be published!"
+description = "Apache Avro tests helper."
+authors = ["Apache Avro team <de...@avro.apache.org>"]
+license = "Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/apache/avro"
+keywords = ["avro", "data", "serialization", "test"]
+categories = ["encoding"]
+documentation = "https://docs.rs/apache-avro-test-helper"
+
[dependencies]
color-backtrace = { default-features = false, version = "0.5.1" }