You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by mg...@apache.org on 2022/02/19 07:41:19 UTC
[avro] branch branch-1.11 updated: AVRO-3407: Test for user metadata in the interop tests (#1552)
This is an automated email from the ASF dual-hosted git repository.
mgrigorov pushed a commit to branch branch-1.11
in repository https://gitbox.apache.org/repos/asf/avro.git
The following commit(s) were added to refs/heads/branch-1.11 by this push:
new 28d6c80 AVRO-3407: Test for user metadata in the interop tests (#1552)
28d6c80 is described below
commit 28d6c802b73948d6f3a8e97441a865bf0115b669
Author: Martin Grigorov <ma...@users.noreply.github.com>
AuthorDate: Sat Feb 19 09:40:22 2022 +0200
AVRO-3407: Test for user metadata in the interop tests (#1552)
* AVRO-3407: Test for user metadata in the interop tests
Signed-off-by: Martin Tzvetanov Grigorov <mg...@apache.org>
* Add user_metadats interop tests in C# (#1554)
Co-authored-by: Zoltan Csizmadia <Cs...@valassis.com>
Co-authored-by: Zoltan Csizmadia <zc...@gmail.com>
Co-authored-by: Zoltan Csizmadia <Cs...@valassis.com>
(cherry picked from commit 7a8576475f776109d060ffdf22e876b754ac3ab4)
---
lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs | 1 +
lang/csharp/src/apache/test/Interop/InteropDataTests.cs | 5 +++++
.../avro/src/main/java/org/apache/avro/util/RandomData.java | 3 +--
.../src/test/java/org/apache/avro/DataFileInteropTest.java | 7 +++----
lang/perl/share/interop-data-generate | 3 +--
lang/perl/xt/interop.t | 13 +++----------
lang/py/avro/test/gen_interop_data.py | 1 +
lang/py/avro/test/test_datafile_interop.py | 7 ++++++-
lang/rust/examples/generate_interop_data.rs | 3 +--
lang/rust/examples/test_interop_data.rs | 13 ++-----------
10 files changed, 24 insertions(+), 32 deletions(-)
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
index 10c06f7..7aa10c0 100644
--- a/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
+++ b/lang/csharp/src/apache/test/Interop/InteropDataGenerator.cs
@@ -86,6 +86,7 @@ namespace Avro.Test.Interop
var codec = Codec.CreateCodecFromString(codecName);
using (var dataFileWriter = DataFileWriter<GenericRecord>.OpenWriter(datumWriter, outputPath, codec))
{
+ dataFileWriter.SetMeta("user_metadata", "someByteArray");
dataFileWriter.Append(record);
}
}
diff --git a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
index 7215df4..4f66a7e 100644
--- a/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
+++ b/lang/csharp/src/apache/test/Interop/InteropDataTests.cs
@@ -48,6 +48,11 @@ namespace Avro.Test.Interop
using(var reader = DataFileReader<GenericRecord>.OpenReader(avroFile))
{
int i = 0;
+ string userMetadata = reader.GetMetaString("user_metadata");
+ if (userMetadata != null)
+ {
+ Assert.AreEqual("someByteArray", userMetadata);
+ }
foreach (var record in reader.NextEntries)
{
i++;
diff --git a/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java b/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
index 12b8a7b..0f364b7 100644
--- a/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
+++ b/lang/java/avro/src/main/java/org/apache/avro/util/RandomData.java
@@ -170,8 +170,7 @@ public class RandomData implements Iterable<Object> {
Schema sch = new Schema.Parser().parse(new File(args[0]));
try (DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>())) {
writer.setCodec(CodecFactory.fromString(args.length >= 4 ? args[3] : "null"));
- writer.setMeta("stringKey", "stringValue");
- writer.setMeta("bytesKey", "bytesValue".getBytes(StandardCharsets.UTF_8));
+ writer.setMeta("user_metadata", "someByteArray".getBytes(StandardCharsets.UTF_8));
writer.create(sch, new File(args[1]));
for (Object datum : new RandomData(sch, Integer.parseInt(args[2]))) {
diff --git a/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java b/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
index 7828532..2ea1e5d 100644
--- a/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
+++ b/lang/java/ipc/src/test/java/org/apache/avro/DataFileInteropTest.java
@@ -86,10 +86,9 @@ public class DataFileInteropTest {
try (DataFileReader<? extends Object> reader = (DataFileReader<? extends Object>) DataFileReader.openReader(f,
provider.get())) {
- // Ignore avro.schema & avro.codec. Some SDKs do not support user metadata.
- if (reader.getMetaKeys().size() > 2) {
- assertEquals("stringValue", reader.getMetaString("stringKey"));
- assertArrayEquals("bytesValue".getBytes(StandardCharsets.UTF_8), reader.getMeta("bytesKey"));
+ byte[] user_metadata = reader.getMeta("user_metadata");
+ if (user_metadata != null) {
+ assertArrayEquals("someByteArray".getBytes(StandardCharsets.UTF_8), user_metadata);
}
int i = 0;
diff --git a/lang/perl/share/interop-data-generate b/lang/perl/share/interop-data-generate
index 5d2cd79..0144e71 100644
--- a/lang/perl/share/interop-data-generate
+++ b/lang/perl/share/interop-data-generate
@@ -55,8 +55,7 @@ my $datum = {
};
my $metadata = {
- stringKey => 'stringValue',
- bytesKey => 'bytesValue'
+ user_metadata => 'someByteArray'
};
while (my ($codec, $enabled) = each(%Avro::DataFile::ValidCodec)) {
diff --git a/lang/perl/xt/interop.t b/lang/perl/xt/interop.t
index d50ca6c..8bc2a84 100644
--- a/lang/perl/xt/interop.t
+++ b/lang/perl/xt/interop.t
@@ -24,11 +24,6 @@ use IO::File;
use_ok 'Avro::DataFile';
use_ok 'Avro::DataFileReader';
-my $expected_metadata = {
- stringKey => 'stringValue',
- bytesKey => 'bytesValue'
-};
-
for my $path (glob '../../build/interop/data/*.avro') {
my $fn = basename($path);
substr($fn, rindex $fn, '.') = '';
@@ -44,12 +39,10 @@ for my $path (glob '../../build/interop/data/*.avro') {
my $reader = Avro::DataFileReader->new(fh => $fh);
my $metadata = $reader->metadata;
- if (exists $metadata->{stringKey}) {
- is($metadata->{stringKey}, $expected_metadata->{stringKey}, "check user metadata: stringKey ");
- }
- if (exists $metadata->{bytesKey}) {
- is($metadata->{bytesKey}, join('', $expected_metadata->{bytesKey}), "check user metadata: bytesKey ");
+ if (exists $metadata->{user_metadata}) {
+ is($metadata->{user_metadata}, 'someByteArray', "check user metadata");
}
+
diag("Succeeded: ${path}");
}
diff --git a/lang/py/avro/test/gen_interop_data.py b/lang/py/avro/test/gen_interop_data.py
index d1f3c0e..1993b01 100644
--- a/lang/py/avro/test/gen_interop_data.py
+++ b/lang/py/avro/test/gen_interop_data.py
@@ -54,6 +54,7 @@ DATUM = {
def gen_data(codec: str, datum_writer: avro.io.DatumWriter, interop_schema: avro.schema.Schema) -> bytes:
with io.BytesIO() as file_, avro.datafile.DataFileWriter(file_, datum_writer, interop_schema, codec=codec) as dfw:
+ dfw.set_meta("user_metadata", b"someByteArray")
dfw.append(DATUM)
dfw.flush()
return file_.getvalue()
diff --git a/lang/py/avro/test/test_datafile_interop.py b/lang/py/avro/test/test_datafile_interop.py
index aeb6c21..d9e4c69 100644
--- a/lang/py/avro/test/test_datafile_interop.py
+++ b/lang/py/avro/test/test_datafile_interop.py
@@ -37,11 +37,16 @@ class TestDataFileInterop(unittest.TestCase):
for filename in _INTEROP_DATA_DIR.iterdir():
self.assertGreater(os.stat(filename).st_size, 0)
base_ext = filename.stem.split("_", 1)
- if len(base_ext) < 2 or base_ext[1] not in avro.codecs.KNOWN_CODECS:
+ if len(base_ext) > 1 and base_ext[1] not in avro.codecs.KNOWN_CODECS:
print(f"SKIPPING {filename} due to an unsupported codec\n")
continue
i = None
with self.subTest(filename=filename), avro.datafile.DataFileReader(filename.open("rb"), avro.io.DatumReader()) as dfr:
+
+ user_metadata = dfr.get_meta("user_metadata")
+ if user_metadata is not None:
+ self.assertEqual(user_metadata, b"someByteArray")
+
for i, datum in enumerate(cast(avro.datafile.DataFileReader, dfr), 1):
self.assertIsNotNone(datum)
self.assertIsNotNone(i)
diff --git a/lang/rust/examples/generate_interop_data.rs b/lang/rust/examples/generate_interop_data.rs
index 3d91140..3bbbb3a 100644
--- a/lang/rust/examples/generate_interop_data.rs
+++ b/lang/rust/examples/generate_interop_data.rs
@@ -101,8 +101,7 @@ fn main() -> anyhow::Result<()> {
}
fn write_user_metadata(writer: &mut Writer<Vec<u8>>) -> anyhow::Result<()> {
- writer.add_user_metadata("stringKey".to_string(), "stringValue")?;
- writer.add_user_metadata("bytesKey".to_string(), b"bytesValue")?;
+ writer.add_user_metadata("user_metadata".to_string(), b"someByteArray")?;
Ok(())
}
diff --git a/lang/rust/examples/test_interop_data.rs b/lang/rust/examples/test_interop_data.rs
index 85d59de..82d2aa1 100644
--- a/lang/rust/examples/test_interop_data.rs
+++ b/lang/rust/examples/test_interop_data.rs
@@ -19,7 +19,8 @@ use apache_avro::Reader;
use std::{collections::HashMap, ffi::OsStr, fs::File};
fn main() -> anyhow::Result<()> {
- let expected_user_metadata: HashMap<String, Vec<u8>> = create_expected_user_metadata();
+ let mut expected_user_metadata: HashMap<String, Vec<u8>> = HashMap::new();
+ expected_user_metadata.insert("user_metadata".to_string(), b"someByteArray".to_vec());
let data_dir = std::fs::read_dir("../../build/interop/data/")
.expect("Unable to list the interop data directory");
@@ -63,16 +64,6 @@ fn main() -> anyhow::Result<()> {
}
}
-fn create_expected_user_metadata() -> HashMap<String, Vec<u8>> {
- let mut user_metadata: HashMap<String, Vec<u8>> = HashMap::new();
- user_metadata.insert(
- "stringKey".to_string(),
- "stringValue".to_string().into_bytes(),
- );
- user_metadata.insert("bytesKey".to_string(), b"bytesValue".to_vec());
- user_metadata
-}
-
fn test_user_metadata(reader: &Reader<&File>, expected_user_metadata: &HashMap<String, Vec<u8>>) {
let user_metadata = reader.user_metadata();
if !user_metadata.is_empty() {