You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by em...@apache.org on 2019/07/05 16:57:27 UTC
[arrow] branch master updated: ARROW-5846: [Java] Create Avro
adapter module and add dependencies
This is an automated email from the ASF dual-hosted git repository.
emkornfield pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new ee58742 ARROW-5846: [Java] Create Avro adapter module and add dependencies
ee58742 is described below
commit ee587427489657bbd5c64a380fa1fb8d59fdda5b
Author: tianchen <ni...@alibaba-inc.com>
AuthorDate: Fri Jul 5 09:56:18 2019 -0700
ARROW-5846: [Java] Create Avro adapter module and add dependencies
Related to [ARROW-5846](https://issues.apache.org/jira/browse/ARROW-5846).
Author: tianchen <ni...@alibaba-inc.com>
Closes #4800 from tianchen92/ARROW-5846 and squashes the following commits:
810784750 <tianchen> fix 2
2c5ff6622 <tianchen> fix
de49b3af8 <tianchen> add test
79c04c879 <tianchen> update maven-compiler-plugin version
d60cc8b89 <tianchen> update version
2a060fbf3 <tianchen> ARROW-5846: Create Avro adapter module and add dependencies
---
java/adapter/avro/pom.xml | 38 +++++++++
.../org/apache/arrow/TestWriteReadAvroRecord.java | 94 ++++++++++++++++++++++
.../avro/src/test/resources/schema/test.avsc | 26 ++++++
java/pom.xml | 1 +
4 files changed, 159 insertions(+)
diff --git a/java/adapter/avro/pom.xml b/java/adapter/avro/pom.xml
new file mode 100644
index 0000000..4e13d57
--- /dev/null
+++ b/java/adapter/avro/pom.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-java-root</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>arrow-avro</artifactId>
+ <name>Arrow AVRO Adapter</name>
+ <url>http://maven.apache.org</url>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>1.9.0</version>
+ </dependency>
+ </dependencies>
+
+</project>
diff --git a/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java b/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java
new file mode 100644
index 0000000..aa5c45a
--- /dev/null
+++ b/java/adapter/avro/src/test/java/org/apache/arrow/TestWriteReadAvroRecord.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.avro.Schema;
+import org.apache.avro.file.DataFileReader;
+import org.apache.avro.file.DataFileWriter;
+import org.apache.avro.generic.GenericData;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.generic.GenericRecord;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.DatumWriter;
+
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+
+public class TestWriteReadAvroRecord {
+
+ @ClassRule
+ public static final TemporaryFolder TMP = new TemporaryFolder();
+
+ @Test
+ public void testWriteAndRead() throws Exception {
+
+ File dataFile = TMP.newFile();
+ Path schemaPath = Paths.get(TestWriteReadAvroRecord.class.getResource("/").getPath(), "schema", "test.avsc");
+ Schema schema = new Schema.Parser().parse(schemaPath.toFile());
+
+ //write data to disk
+ GenericRecord user1 = new GenericData.Record(schema);
+ user1.put("name", "Alyssa");
+ user1.put("favorite_number", 256);
+
+ GenericRecord user2 = new GenericData.Record(schema);
+ user2.put("name", "Ben");
+ user2.put("favorite_number", 7);
+ user2.put("favorite_color", "red");
+
+ DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema);
+ DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
+ dataFileWriter.create(schema, dataFile);
+ dataFileWriter.append(user1);
+ dataFileWriter.append(user2);
+ dataFileWriter.close();
+
+ //read data from disk
+ DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>(schema);
+ DataFileReader<GenericRecord>
+ dataFileReader = new DataFileReader<GenericRecord>(dataFile, datumReader);
+ List<GenericRecord> result = new ArrayList<>();
+ while (dataFileReader.hasNext()) {
+ GenericRecord user = dataFileReader.next();
+ result.add(user);
+ }
+
+ assertEquals(2, result.size());
+ GenericRecord deUser1 = result.get(0);
+ assertEquals("Alyssa", deUser1.get("name").toString());
+ assertEquals(256, deUser1.get("favorite_number"));
+ assertEquals(null, deUser1.get("favorite_color"));
+
+ GenericRecord deUser2 = result.get(1);
+ assertEquals("Ben", deUser2.get("name").toString());
+ assertEquals(7, deUser2.get("favorite_number"));
+ assertEquals("red", deUser2.get("favorite_color").toString());
+ }
+
+}
diff --git a/java/adapter/avro/src/test/resources/schema/test.avsc b/java/adapter/avro/src/test/resources/schema/test.avsc
new file mode 100644
index 0000000..15fdd76
--- /dev/null
+++ b/java/adapter/avro/src/test/resources/schema/test.avsc
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+{"namespace": "org.apache.arrow.avro",
+ "type": "record",
+ "name": "User",
+ "fields": [
+ {"name": "name", "type": "string"},
+ {"name": "favorite_number", "type": ["int", "null"]},
+ {"name": "favorite_color", "type": ["string", "null"]}
+ ]
+}
diff --git a/java/pom.xml b/java/pom.xml
index 1358f5f..29d602d 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -655,6 +655,7 @@
<module>flight</module>
<module>performance</module>
<module>algorithm</module>
+ <module>adapter/avro</module>
</modules>
<profiles>