You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/05/11 20:49:38 UTC
[1/2] arrow git commit: ARROW-482 [Java] Exposing custom field
metadata
Repository: arrow
Updated Branches:
refs/heads/master a8338f1f8 -> 010bd2245
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
index 27879ef..9678423 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/stream/MessageSerializerTest.java
@@ -29,6 +29,7 @@ import java.nio.channels.Channels;
import java.util.Collections;
import java.util.List;
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.file.ArrowBlock;
@@ -40,13 +41,12 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
-import org.junit.Test;
import org.junit.Rule;
+import org.junit.Test;
import org.junit.rules.ExpectedException;
-import io.netty.buffer.ArrowBuf;
-
public class MessageSerializerTest {
public static ArrowBuf buf(BufferAllocator alloc, byte[] bytes) {
@@ -79,7 +79,7 @@ public class MessageSerializerTest {
@Test
public void testSchemaDictionaryMessageSerialization() throws IOException {
DictionaryEncoding dictionary = new DictionaryEncoding(9L, false, new ArrowType.Int(8, true));
- Field field = new Field("test", true, ArrowType.Utf8.INSTANCE, dictionary, null);
+ Field field = new Field("test", new FieldType(true, ArrowType.Utf8.INSTANCE, dictionary, null), null);
Schema schema = new Schema(Collections.singletonList(field));
ByteArrayOutputStream out = new ByteArrayOutputStream();
long size = MessageSerializer.serialize(new WriteChannel(Channels.newChannel(out)), schema);
@@ -130,7 +130,7 @@ public class MessageSerializerTest {
public static Schema testSchema() {
return new Schema(asList(new Field(
- "testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+ "testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())));
}
// Verifies batch contents matching test schema.
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
index 56fa73e..84cc107 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/types/pojo/TestSchema.java
@@ -47,7 +47,7 @@ import org.junit.Test;
public class TestSchema {
private static Field field(String name, boolean nullable, ArrowType type, Field... children) {
- return new Field(name, nullable, type, asList(children));
+ return new Field(name, new FieldType(nullable, type, null, null), asList(children));
}
private static Field field(String name, ArrowType type, Field... children) {
[2/2] arrow git commit: ARROW-482 [Java] Exposing custom field
metadata
Posted by we...@apache.org.
ARROW-482 [Java] Exposing custom field metadata
I added the metadata to the `FieldType` object. It doesn't necessarily seem the right place for it, but it minimizes having to change method signatures to pass around the metadata. I also tried to standardize vector constructors on `FieldType` to ensure it's passed correctly.
Author: Emilio Lahr-Vivaz <el...@ccri.com>
Closes #611 from elahrvivaz/ARROW-482 and squashes the following commits:
63a76e0 [Emilio Lahr-Vivaz] ARROW-482 [Java] Exposing custom field metadata
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/010bd224
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/010bd224
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/010bd224
Branch: refs/heads/master
Commit: 010bd224515b798e2e2429d3dfaa51ad8223bdbc
Parents: a8338f1
Author: Emilio Lahr-Vivaz <el...@ccri.com>
Authored: Thu May 11 16:49:33 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Thu May 11 16:49:33 2017 -0400
----------------------------------------------------------------------
.../arrow/tools/ArrowFileTestFixtures.java | 2 +-
.../org/apache/arrow/tools/EchoServerTest.java | 10 +-
.../org/apache/arrow/tools/TestIntegration.java | 4 +-
.../src/main/codegen/templates/NullReader.java | 3 +-
.../codegen/templates/NullableValueVectors.java | 4 +-
.../src/main/codegen/templates/UnionVector.java | 6 +-
.../org/apache/arrow/vector/ZeroVector.java | 3 +-
.../vector/complex/FixedSizeListVector.java | 44 +++++----
.../apache/arrow/vector/complex/ListVector.java | 34 ++++---
.../apache/arrow/vector/complex/MapVector.java | 36 +++++--
.../arrow/vector/complex/NullableMapVector.java | 36 ++++---
.../vector/dictionary/DictionaryEncoder.java | 6 +-
.../apache/arrow/vector/file/ArrowReader.java | 5 +-
.../apache/arrow/vector/file/ArrowWriter.java | 3 +-
.../org/apache/arrow/vector/types/Types.java | 7 +-
.../apache/arrow/vector/types/pojo/Field.java | 83 +++++++++++-----
.../arrow/vector/types/pojo/FieldType.java | 15 ++-
.../arrow/vector/TestFixedSizeListVector.java | 6 +-
.../org/apache/arrow/vector/TestListVector.java | 4 +-
.../apache/arrow/vector/TestVectorReAlloc.java | 4 +-
.../arrow/vector/TestVectorUnloadLoad.java | 9 +-
.../complex/impl/TestPromotableWriter.java | 4 +-
.../complex/writer/TestComplexWriter.java | 43 ++++-----
.../apache/arrow/vector/file/TestArrowFile.java | 99 +++++++++++++++++---
.../arrow/vector/file/TestArrowFooter.java | 3 +-
.../vector/file/TestArrowReaderWriter.java | 3 +-
.../arrow/vector/file/json/TestJSONFile.java | 8 +-
.../apache/arrow/vector/pojo/TestConvert.java | 33 +++----
.../vector/stream/MessageSerializerTest.java | 10 +-
.../arrow/vector/types/pojo/TestSchema.java | 2 +-
30 files changed, 355 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
----------------------------------------------------------------------
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
index 34c93ed..6d9a6c1 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/ArrowFileTestFixtures.java
@@ -99,7 +99,7 @@ public class ArrowFileTestFixtures {
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", vectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), testInFile);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
----------------------------------------------------------------------
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
index 7cca339..d7f6388 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/EchoServerTest.java
@@ -137,8 +137,8 @@ public class EchoServerTest {
BufferAllocator alloc = new RootAllocator(Long.MAX_VALUE);
Field field = new Field(
- "testField", true,
- new ArrowType.Int(8, true),
+ "testField",
+ new FieldType(true, new ArrowType.Int(8, true), null, null),
Collections.<Field>emptyList());
NullableTinyIntVector vector =
new NullableTinyIntVector("testField", FieldType.nullable(TINYINT.getType()), alloc);
@@ -161,7 +161,7 @@ public class EchoServerTest {
NullableIntVector writeVector =
new NullableIntVector(
"varchar",
- new FieldType(true, MinorType.INT.getType(), writeEncoding),
+ new FieldType(true, MinorType.INT.getType(), writeEncoding, null),
allocator);
NullableVarCharVector writeDictionaryVector =
new NullableVarCharVector(
@@ -237,7 +237,7 @@ public class EchoServerTest {
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
NullableVarCharVector writeDictionaryVector =
new NullableVarCharVector("dictionary", FieldType.nullable(VARCHAR.getType()), allocator);
- ListVector writeVector = new ListVector("list", allocator, null, null)) {
+ ListVector writeVector = ListVector.empty("list", allocator)) {
// data being written:
// [['foo', 'bar'], ['foo'], ['bar']] -> [[0, 1], [0], [1]]
@@ -247,7 +247,7 @@ public class EchoServerTest {
writeDictionaryVector.getMutator().set(1, "bar".getBytes(StandardCharsets.UTF_8));
writeDictionaryVector.getMutator().setValueCount(2);
- writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding));
+ writeVector.addOrGetVector(new FieldType(true, MinorType.INT.getType(), writeEncoding, null));
writeVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(writeVector);
listWriter.startList();
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
----------------------------------------------------------------------
diff --git a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
index 7d9a419..4b840df 100644
--- a/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
+++ b/java/tools/src/test/java/org/apache/arrow/tools/TestIntegration.java
@@ -76,7 +76,7 @@ public class TestIntegration {
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", vectorAllocator)) {
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
Float8Writer floatWriter = rootWriter.float8("float");
@@ -95,7 +95,7 @@ public class TestIntegration {
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0,
Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", vectorAllocator)) {
writeData(count, parent);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/codegen/templates/NullReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullReader.java b/java/vector/src/main/codegen/templates/NullReader.java
index 7c75b3a..284a8d3 100644
--- a/java/vector/src/main/codegen/templates/NullReader.java
+++ b/java/vector/src/main/codegen/templates/NullReader.java
@@ -55,10 +55,9 @@ public class NullReader extends AbstractBaseReader implements FieldReader{
return type;
}
-
@Override
public Field getField() {
- return new Field("", true, new Null(), null);
+ return new Field("", FieldType.nullable(new Null()), null);
}
public void copyAsValue(MapWriter writer) {}
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 76d2bad..ed2418e 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -66,11 +66,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
private final int scale;
public ${className}(String name, BufferAllocator allocator, int precision, int scale) {
- this(name, new FieldType(true, new Decimal(precision, scale), null), allocator);
+ this(name, FieldType.nullable(new Decimal(precision, scale)), allocator);
}
<#else>
public ${className}(String name, BufferAllocator allocator) {
- this(name, new FieldType(true, org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType(), null), allocator);
+ this(name, FieldType.nullable(org.apache.arrow.vector.types.Types.MinorType.${minor.class?upper_case}.getType()), allocator);
}
</#if>
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 9d5dee5..ec42b12 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -77,7 +77,7 @@ public class UnionVector implements FieldVector {
public UnionVector(String name, BufferAllocator allocator, CallBack callBack) {
this.name = name;
this.allocator = allocator;
- this.internalMap = new MapVector("internal", allocator, callBack);
+ this.internalMap = new MapVector("internal", allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
this.typeVector = new UInt1Vector("types", allocator);
this.callBack = callBack;
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(typeVector));
@@ -125,7 +125,7 @@ public class UnionVector implements FieldVector {
}
private FieldType fieldType(MinorType type) {
- return new FieldType(true, type.getType(), null);
+ return FieldType.nullable(type.getType());
}
private <T extends FieldVector> T addOrGet(MinorType minorType, Class<T> c) {
@@ -250,7 +250,7 @@ public class UnionVector implements FieldVector {
typeIds[childFields.size()] = v.getMinorType().ordinal();
childFields.add(v.getField());
}
- return new Field(name, true, new ArrowType.Union(Sparse, typeIds), childFields);
+ return new Field(name, FieldType.nullable(new ArrowType.Union(Sparse, typeIds)), childFields);
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index e48788c..01e22f2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -29,6 +29,7 @@ import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.ArrowType.Null;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;
@@ -98,7 +99,7 @@ public class ZeroVector implements FieldVector {
@Override
public Field getField() {
- return new Field(name, true, new Null(), null);
+ return new Field(name, FieldType.nullable(new Null()), null);
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 0dceeed..ee22fbe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -26,6 +26,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
@@ -48,18 +53,17 @@ import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
public class FixedSizeListVector extends BaseValueVector implements FieldVector, PromotableVector {
+ public static FixedSizeListVector empty(String name, int size, BufferAllocator allocator) {
+ FieldType fieldType = FieldType.nullable(new ArrowType.FixedSizeList(size));
+ return new FixedSizeListVector(name, allocator, fieldType, null);
+ }
+
private FieldVector vector;
private final BitVector bits;
private final int listSize;
- private final DictionaryEncoding dictionary;
+ private final FieldType fieldType;
private final List<BufferBacked> innerVectors;
private UnionFixedSizeListReader reader;
@@ -67,17 +71,26 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
private Mutator mutator = new Mutator();
private Accessor accessor = new Accessor();
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public FixedSizeListVector(String name,
BufferAllocator allocator,
int listSize,
DictionaryEncoding dictionary,
CallBack schemaChangeCallback) {
+ this(name, allocator, new FieldType(true, new ArrowType.FixedSizeList(listSize), dictionary), schemaChangeCallback);
+ }
+
+ public FixedSizeListVector(String name,
+ BufferAllocator allocator,
+ FieldType fieldType,
+ CallBack schemaChangeCallback) {
super(name, allocator);
- Preconditions.checkArgument(listSize > 0, "list size must be positive");
this.bits = new BitVector("$bits$", allocator);
this.vector = ZeroVector.INSTANCE;
- this.listSize = listSize;
- this.dictionary = dictionary;
+ this.fieldType = fieldType;
+ this.listSize = ((ArrowType.FixedSizeList) fieldType.getType()).getListSize();
+ Preconditions.checkArgument(listSize > 0, "list size must be positive");
this.innerVectors = Collections.singletonList((BufferBacked) bits);
this.reader = new UnionFixedSizeListReader(this);
}
@@ -85,7 +98,7 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
@Override
public Field getField() {
List<Field> children = ImmutableList.of(getDataVector().getField());
- return new Field(name, true, new ArrowType.FixedSizeList(listSize), children);
+ return new Field(name, fieldType, children);
}
@Override
@@ -103,8 +116,7 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
throw new IllegalArgumentException("Lists have only one child. Found: " + children);
}
Field field = children.get(0);
- FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
- AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(type);
+ AddOrGetResult<FieldVector> addOrGetVector = addOrGetVector(field.getFieldType());
if (!addOrGetVector.isCreated()) {
throw new IllegalArgumentException("Child vector already existed: " + addOrGetVector.getVector());
}
@@ -348,14 +360,12 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
TransferPair pairs[] = new TransferPair[2];
public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
- this(new FixedSizeListVector(name, allocator, listSize, dictionary, callBack));
+ this(new FixedSizeListVector(name, allocator, fieldType, callBack));
}
public TransferImpl(FixedSizeListVector to) {
this.to = to;
- Field field = vector.getField();
- FieldType type = new FieldType(field.isNullable(), field.getType(), field.getDictionary());
- to.addOrGetVector(type);
+ to.addOrGetVector(vector.getField().getFieldType());
pairs[0] = bits.makeTransferPair(to.bits);
pairs[1] = getDataVector().makeTransferPair(to.getDataVector());
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 9392afb..c2f5576 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,12 +18,17 @@
******************************************************************************/
package org.apache.arrow.vector.complex;
+import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.AddOrGetResult;
@@ -41,6 +46,7 @@ import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
@@ -48,13 +54,12 @@ import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
+ public static ListVector empty(String name, BufferAllocator allocator) {
+ return new ListVector(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), null);
+ }
+
final UInt4Vector offsets;
final BitVector bits;
private final List<BufferBacked> innerVectors;
@@ -62,19 +67,27 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
private Accessor accessor = new Accessor();
private UnionListReader reader;
private CallBack callBack;
- private final DictionaryEncoding dictionary;
+ private final FieldType fieldType;
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
- this(name, allocator, null, callBack);
+ this(name, allocator, FieldType.nullable(ArrowType.List.INSTANCE), callBack);
}
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public ListVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
+ this(name, allocator, new FieldType(true, ArrowType.List.INSTANCE, dictionary, null), callBack);
+ }
+
+ public ListVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
super(name, allocator, callBack);
this.bits = new BitVector("$bits$", allocator);
this.offsets = getOffsetVector();
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits, offsets));
this.reader = new UnionListReader(this);
- this.dictionary = dictionary;
+ this.fieldType = checkNotNull(fieldType);
this.callBack = callBack;
}
@@ -168,7 +181,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
TransferPair pairs[] = new TransferPair[3];
public TransferImpl(String name, BufferAllocator allocator, CallBack callBack) {
- this(new ListVector(name, allocator, dictionary, callBack));
+ this(new ListVector(name, allocator, fieldType, callBack));
}
public TransferImpl(ListVector to) {
@@ -266,8 +279,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
@Override
public Field getField() {
- return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
- ImmutableList.of(getDataVector().getField()));
+ return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index 997a6a3..bdd30f8 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -17,6 +17,8 @@
*/
package org.apache.arrow.vector.complex;
+import static com.google.common.base.Preconditions.checkNotNull;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
@@ -25,6 +27,10 @@ import java.util.Map;
import javax.annotation.Nullable;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BaseValueVector;
import org.apache.arrow.vector.FieldVector;
@@ -33,26 +39,36 @@ import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.ComplexHolder;
import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
+import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
public class MapVector extends AbstractMapVector {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+ public static MapVector empty(String name, BufferAllocator allocator) {
+ FieldType fieldType = new FieldType(false, ArrowType.Struct.INSTANCE, null, null);
+ return new MapVector(name, allocator, fieldType, null);
+ }
+
private final SingleMapReaderImpl reader = new SingleMapReaderImpl(this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
+ protected final FieldType fieldType;
public int valueCount;
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public MapVector(String name, BufferAllocator allocator, CallBack callBack) {
+ this(name, allocator, new FieldType(false, ArrowType.Struct.INSTANCE, null, null), callBack);
+ }
+
+ public MapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
super(name, allocator, callBack);
+ this.fieldType = checkNotNull(fieldType);
}
@Override
@@ -119,7 +135,7 @@ public class MapVector extends AbstractMapVector {
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
- return new MapTransferPair(this, new MapVector(name, allocator, callBack), false);
+ return new MapTransferPair(this, new MapVector(name, allocator, fieldType, callBack), false);
}
@Override
@@ -129,7 +145,7 @@ public class MapVector extends AbstractMapVector {
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new MapTransferPair(this, new MapVector(ref, allocator, callBack), false);
+ return new MapTransferPair(this, new MapVector(ref, allocator, fieldType, callBack), false);
}
protected static class MapTransferPair implements TransferPair{
@@ -212,8 +228,8 @@ public class MapVector extends AbstractMapVector {
@Override
public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
return Ints.compare(
- Preconditions.checkNotNull(left).getValueCapacity(),
- Preconditions.checkNotNull(right).getValueCapacity()
+ checkNotNull(left).getValueCapacity(),
+ checkNotNull(right).getValueCapacity()
);
}
};
@@ -294,7 +310,7 @@ public class MapVector extends AbstractMapVector {
for (ValueVector child : getChildren()) {
children.add(child.getField());
}
- return new Field(name, false, Struct.INSTANCE, children);
+ return new Field(name, fieldType, children);
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 6456efb..f1f8640 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -23,6 +23,9 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BaseDataValueVector;
import org.apache.arrow.vector.BitVector;
@@ -32,39 +35,48 @@ import org.apache.arrow.vector.NullableVectorDefinitionSetter;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.impl.NullableMapReaderImpl;
import org.apache.arrow.vector.complex.impl.NullableMapWriter;
-import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.ComplexHolder;
import org.apache.arrow.vector.schema.ArrowFieldNode;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
public class NullableMapVector extends MapVector implements FieldVector {
+ public static NullableMapVector empty(String name, BufferAllocator allocator) {
+ FieldType fieldType = FieldType.nullable(Struct.INSTANCE);
+ return new NullableMapVector(name, allocator, fieldType, null);
+ }
+
private final NullableMapReaderImpl reader = new NullableMapReaderImpl(this);
private final NullableMapWriter writer = new NullableMapWriter(this);
protected final BitVector bits;
private final List<BufferBacked> innerVectors;
- private final DictionaryEncoding dictionary;
private final Accessor accessor;
private final Mutator mutator;
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public NullableMapVector(String name, BufferAllocator allocator, CallBack callBack) {
- this(name, allocator, null, callBack);
+ this(name, allocator, FieldType.nullable(ArrowType.Struct.INSTANCE), callBack);
}
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public NullableMapVector(String name, BufferAllocator allocator, DictionaryEncoding dictionary, CallBack callBack) {
- super(name, checkNotNull(allocator), callBack);
+ this(name, allocator, new FieldType(true, ArrowType.Struct.INSTANCE, dictionary, null), callBack);
+ }
+
+ public NullableMapVector(String name, BufferAllocator allocator, FieldType fieldType, CallBack callBack) {
+ super(name, checkNotNull(allocator), fieldType, callBack);
this.bits = new BitVector("$bits$", allocator);
this.innerVectors = Collections.unmodifiableList(Arrays.<BufferBacked>asList(bits));
- this.dictionary = dictionary;
this.accessor = new Accessor();
this.mutator = new Mutator();
}
@@ -96,7 +108,7 @@ public class NullableMapVector extends MapVector implements FieldVector {
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, dictionary, null), false);
+ return new NullableMapTransferPair(this, new NullableMapVector(name, allocator, fieldType, null), false);
}
@Override
@@ -106,12 +118,12 @@ public class NullableMapVector extends MapVector implements FieldVector {
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, null), false);
+ return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, null), false);
}
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator, CallBack callBack) {
- return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, dictionary, callBack), false);
+ return new NullableMapTransferPair(this, new NullableMapVector(ref, allocator, fieldType, callBack), false);
}
protected class NullableMapTransferPair extends MapTransferPair {
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index 0666bc4..0f49ce6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -29,6 +29,7 @@ import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.TransferPair;
public class DictionaryEncoder {
@@ -53,8 +54,9 @@ public class DictionaryEncoder {
}
Field valueField = vector.getField();
- Field indexField = new Field(valueField.getName(), valueField.isNullable(),
- dictionary.getEncoding().getIndexType(), dictionary.getEncoding(), null);
+ FieldType indexFieldType = new FieldType(valueField.isNullable(), dictionary.getEncoding().getIndexType(),
+ dictionary.getEncoding(), valueField.getMetadata());
+ Field indexField = new Field(valueField.getName(), indexFieldType, null);
// vector to hold our indices (dictionary encoded values)
FieldVector indices = indexField.createVector(vector.getAllocator());
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
index 1d33913..f000885 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowReader.java
@@ -40,6 +40,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
public abstract class ArrowReader<T extends ReadChannel> implements DictionaryProvider, AutoCloseable {
@@ -192,13 +193,13 @@ public abstract class ArrowReader<T extends ReadChannel> implements DictionaryPr
// get existing or create dictionary vector
if (!dictionaries.containsKey(encoding.getId())) {
// create a new dictionary vector for the values
- Field dictionaryField = new Field(field.getName(), field.isNullable(), field.getType(), null, children);
+ Field dictionaryField = new Field(field.getName(), new FieldType(field.isNullable(), field.getType(), null, null), children);
FieldVector dictionaryVector = dictionaryField.createVector(allocator);
dictionaries.put(encoding.getId(), new Dictionary(dictionaryVector, encoding));
}
}
- return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
+ return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
}
private void load(ArrowDictionaryBatch dictionaryBatch) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
index 1716287..35b44fb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/ArrowWriter.java
@@ -36,6 +36,7 @@ import org.apache.arrow.vector.stream.MessageSerializer;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +118,7 @@ public abstract class ArrowWriter implements AutoCloseable {
}
}
- return new Field(field.getName(), field.isNullable(), type, encoding, updatedChildren);
+ return new Field(field.getName(), new FieldType(field.isNullable(), type, encoding, field.getMetadata()), updatedChildren);
}
public void start() throws IOException {
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index d5076d8..2a0e47b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -122,7 +122,7 @@ public class Types {
MAP(Struct.INSTANCE) {
@Override
public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
- return new NullableMapVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
+ return new NullableMapVector(name, allocator, fieldType, schemaChangeCallback);
}
@Override
@@ -430,7 +430,7 @@ public class Types {
LIST(List.INSTANCE) {
@Override
public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
- return new ListVector(name, allocator, fieldType.getDictionary(), schemaChangeCallback);
+ return new ListVector(name, allocator, fieldType, schemaChangeCallback);
}
@Override
@@ -446,8 +446,7 @@ public class Types {
@Override
public FieldVector getNewVector(String name, FieldType fieldType, BufferAllocator allocator, CallBack schemaChangeCallback) {
- int size = ((FixedSizeList)fieldType.getType()).getListSize();
- return new FixedSizeListVector(name, allocator, size, fieldType.getDictionary(), schemaChangeCallback);
+ return new FixedSizeListVector(name, allocator, fieldType, schemaChangeCallback);
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 05eb9cd..a8f2ae5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -21,15 +21,12 @@ package org.apache.arrow.vector.types.pojo;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.schema.TypeLayout;
-import org.apache.arrow.vector.schema.VectorLayout;
-import org.apache.arrow.vector.types.pojo.ArrowType.Int;
-
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -37,8 +34,16 @@ import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.KeyValue;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.schema.TypeLayout;
+import org.apache.arrow.vector.schema.VectorLayout;
+import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+
public class Field {
public static Field nullablePrimitive(String name, ArrowType.PrimitiveType type) {
@@ -46,7 +51,7 @@ public class Field {
}
public static Field nullable(String name, ArrowType type) {
- return new Field(name, true, type, null, null);
+ return new Field(name, FieldType.nullable(type), null);
}
private final String name;
@@ -61,28 +66,32 @@ public class Field {
@JsonProperty("type") ArrowType type,
@JsonProperty("dictionary") DictionaryEncoding dictionary,
@JsonProperty("children") List<Field> children,
- @JsonProperty("typeLayout") TypeLayout typeLayout) {
- this(name, new FieldType(nullable, type, dictionary), children, typeLayout);
+ @JsonProperty("typeLayout") TypeLayout typeLayout,
+ @JsonProperty("metadata") Map<String, String> metadata) {
+ this(name, new FieldType(nullable, type, dictionary, metadata), children, typeLayout);
}
private Field(String name, FieldType fieldType, List<Field> children, TypeLayout typeLayout) {
- super();
this.name = name;
this.fieldType = checkNotNull(fieldType);
- this.children = children == null ? ImmutableList.<Field>of() : children;
+ this.children = children == null ? ImmutableList.<Field>of() : ImmutableList.copyOf(children);
this.typeLayout = checkNotNull(typeLayout);
}
- public Field(String name, FieldType fieldType, List<Field> children) {
- this(name, fieldType, children, TypeLayout.getTypeLayout(fieldType.getType()));
- }
-
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public Field(String name, boolean nullable, ArrowType type, List<Field> children) {
- this(name, nullable, type, null, children);
+ this(name, new FieldType(nullable, type, null, null), children);
}
+ // deprecated, use FieldType or static constructor instead
+ @Deprecated
public Field(String name, boolean nullable, ArrowType type, DictionaryEncoding dictionary, List<Field> children) {
- this(name, new FieldType(nullable, type, dictionary), children);
+ this(name, new FieldType(nullable, type, dictionary, null), children);
+ }
+
+ public Field(String name, FieldType fieldType, List<Field> children) {
+ this(name, fieldType, children, fieldType == null ? null : TypeLayout.getTypeLayout(fieldType.getType()));
}
public FieldVector createVector(BufferAllocator allocator) {
@@ -114,7 +123,14 @@ public class Field {
childrenBuilder.add(convertField(field.children(i)));
}
List<Field> children = childrenBuilder.build();
- return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()));
+ ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
+ for (int i = 0; i < field.customMetadataLength(); i++) {
+ KeyValue kv = field.customMetadata(i);
+ String key = kv.key(), value = kv.value();
+ metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
+ }
+ Map<String, String> metadata = metadataBuilder.build();
+ return new Field(name, nullable, type, dictionary, children, new TypeLayout(layout.build()), metadata);
}
public void validate() {
@@ -147,7 +163,19 @@ public class Field {
VectorLayout vectorLayout = typeLayout.getVectors().get(i);
buffersData[i] = vectorLayout.writeTo(builder);
}
- int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
+ int layoutOffset = org.apache.arrow.flatbuf.Field.createLayoutVector(builder, buffersData);
+ int[] metadataOffsets = new int[getMetadata().size()];
+ Iterator<Entry<String, String>> metadataIterator = getMetadata().entrySet().iterator();
+ for (int i = 0; i < metadataOffsets.length; i ++) {
+ Entry<String, String> kv = metadataIterator.next();
+ int keyOffset = builder.createString(kv.getKey());
+ int valueOffset = builder.createString(kv.getValue());
+ KeyValue.startKeyValue(builder);
+ KeyValue.addKey(builder, keyOffset);
+ KeyValue.addValue(builder, valueOffset);
+ metadataOffsets[i] = KeyValue.endKeyValue(builder);
+ }
+ int metadataOffset = org.apache.arrow.flatbuf.Field.createCustomMetadataVector(builder, metadataOffsets);
org.apache.arrow.flatbuf.Field.startField(builder);
if (name != null) {
org.apache.arrow.flatbuf.Field.addName(builder, nameOffset);
@@ -157,6 +185,7 @@ public class Field {
org.apache.arrow.flatbuf.Field.addType(builder, typeOffset);
org.apache.arrow.flatbuf.Field.addChildren(builder, childrenOffset);
org.apache.arrow.flatbuf.Field.addLayout(builder, layoutOffset);
+ org.apache.arrow.flatbuf.Field.addCustomMetadata(builder, metadataOffset);
if (dictionary != null) {
org.apache.arrow.flatbuf.Field.addDictionary(builder, dictionaryOffset);
}
@@ -193,9 +222,14 @@ public class Field {
return typeLayout;
}
+ @JsonInclude(Include.NON_EMPTY)
+ public Map<String, String> getMetadata() {
+ return fieldType.getMetadata();
+ }
+
@Override
public int hashCode() {
- return Objects.hash(name, isNullable(), getType(), getDictionary(), children);
+ return Objects.hash(name, isNullable(), getType(), getDictionary(), getMetadata(), children);
}
@Override
@@ -205,10 +239,11 @@ public class Field {
}
Field that = (Field) obj;
return Objects.equals(this.name, that.name) &&
- Objects.equals(this.isNullable(), that.isNullable()) &&
- Objects.equals(this.getType(), that.getType()) &&
- Objects.equals(this.getDictionary(), that.getDictionary()) &&
- Objects.equals(this.children, that.children);
+ Objects.equals(this.isNullable(), that.isNullable()) &&
+ Objects.equals(this.getType(), that.getType()) &&
+ Objects.equals(this.getDictionary(), that.getDictionary()) &&
+ Objects.equals(this.getMetadata(), that.getMetadata()) &&
+ Objects.equals(this.children, that.children);
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
index fe99e63..c8fc689 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
@@ -19,6 +19,10 @@ package org.apache.arrow.vector.types.pojo;
import static com.google.common.base.Preconditions.checkNotNull;
+import java.util.Map;
+
+import com.google.common.collect.ImmutableMap;
+
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.types.Types;
@@ -28,18 +32,24 @@ import org.apache.arrow.vector.util.CallBack;
public class FieldType {
public static FieldType nullable(ArrowType type) {
- return new FieldType(true, type, null);
+ return new FieldType(true, type, null, null);
}
private final boolean nullable;
private final ArrowType type;
private final DictionaryEncoding dictionary;
+ private final Map<String, String> metadata;
public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary) {
+ this(nullable, type, dictionary, null);
+ }
+
+ public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary, Map<String, String> metadata) {
super();
this.nullable = nullable;
this.type = checkNotNull(type);
this.dictionary = dictionary;
+ this.metadata = metadata == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(metadata);
}
public boolean isNullable() {
@@ -51,6 +61,9 @@ public class FieldType {
public DictionaryEncoding getDictionary() {
return dictionary;
}
+ public Map<String, String> getMetadata() {
+ return metadata;
+ }
public FieldVector createNewSingleVector(String name, BufferAllocator allocator, CallBack schemaCallBack) {
MinorType minorType = Types.getMinorTypeForArrowType(type);
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index cfb7b3d..f35ab54 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -49,7 +49,7 @@ public class TestFixedSizeListVector {
@Test
public void testIntType() {
- try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+ try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
NullableIntVector nested = (NullableIntVector) vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType())).getVector();
NullableIntVector.Mutator mutator = nested.getMutator();
vector.allocateNew();
@@ -77,7 +77,7 @@ public class TestFixedSizeListVector {
@Test
public void testFloatTypeNullable() {
- try (FixedSizeListVector vector = new FixedSizeListVector("list", allocator, 2, null, null)) {
+ try (FixedSizeListVector vector = FixedSizeListVector.empty("list", 2, allocator)) {
NullableFloat4Vector nested = (NullableFloat4Vector) vector.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
NullableFloat4Vector.Mutator mutator = nested.getMutator();
vector.allocateNew();
@@ -112,7 +112,7 @@ public class TestFixedSizeListVector {
@Test
public void testNestedInList() {
- try (ListVector vector = new ListVector("list", allocator, null, null)) {
+ try (ListVector vector = ListVector.empty("list", allocator)) {
ListVector.Mutator mutator = vector.getMutator();
FixedSizeListVector tuples = (FixedSizeListVector) vector.addOrGetVector(FieldType.nullable(new ArrowType.FixedSizeList(2))).getVector();
FixedSizeListVector.Mutator tupleMutator = tuples.getMutator();
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
index 18d93b6..11be329 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestListVector.java
@@ -42,8 +42,8 @@ public class TestListVector {
@Test
public void testCopyFrom() throws Exception {
- try (ListVector inVector = new ListVector("input", allocator, null, null);
- ListVector outVector = new ListVector("output", allocator, null, null)) {
+ try (ListVector inVector = ListVector.empty("input", allocator);
+ ListVector outVector = ListVector.empty("output", allocator)) {
UnionListWriter writer = inVector.getWriter();
writer.allocate();
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 40c7bc5..da9cb00 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -123,7 +123,7 @@ public class TestVectorReAlloc {
@Test
public void testListType() {
- try (final ListVector vector = new ListVector("", allocator, null)) {
+ try (final ListVector vector = ListVector.empty("", allocator)) {
vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
vector.setInitialCapacity(512);
@@ -146,7 +146,7 @@ public class TestVectorReAlloc {
@Test
public void testMapType() {
- try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ try (final NullableMapVector vector = NullableMapVector.empty("", allocator)) {
vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
vector.setInitialCapacity(512);
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
index 372bcf0..6fb559c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorUnloadLoad.java
@@ -42,6 +42,7 @@ import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.schema.ArrowRecordBatch;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -58,7 +59,7 @@ public class TestVectorUnloadLoad {
try (
BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
// write some data
ComplexWriter writer = new ComplexWriterImpl("root", parent);
@@ -106,7 +107,7 @@ public class TestVectorUnloadLoad {
Schema schema;
try (
BufferAllocator originalVectorsAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorsAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", originalVectorsAllocator)) {
// write some data
ComplexWriter writer = new ComplexWriterImpl("root", parent);
@@ -182,8 +183,8 @@ public class TestVectorUnloadLoad {
@Test
public void testLoadEmptyValidityBuffer() throws IOException {
Schema schema = new Schema(asList(
- new Field("intDefined", true, new ArrowType.Int(32, true), Collections.<Field>emptyList()),
- new Field("intNull", true, new ArrowType.Int(32, true), Collections.<Field>emptyList())
+ new Field("intDefined", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList()),
+ new Field("intNull", FieldType.nullable(new ArrowType.Int(32, true)), Collections.<Field>emptyList())
));
int count = 10;
ArrowBuf validity = allocator.buffer(10).slice(0, 0);
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
index 65b193c..e826fa5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/impl/TestPromotableWriter.java
@@ -27,8 +27,10 @@ import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.NullableMapVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -51,7 +53,7 @@ public class TestPromotableWriter {
@Test
public void testPromoteToUnion() throws Exception {
- try (final MapVector container = new MapVector(EMPTY_SCHEMA_PATH, allocator, null);
+ try (final MapVector container = MapVector.empty(EMPTY_SCHEMA_PATH, allocator);
final NullableMapVector v = container.addOrGetMap("test");
final PromotableWriter writer = new PromotableWriter(v, container)) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index aba65db..1613936 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -17,16 +17,13 @@
*/
package org.apache.arrow.vector.complex.writer;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.SchemaChangeCallBack;
@@ -48,9 +45,11 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
+import org.apache.arrow.vector.types.pojo.ArrowType.Struct;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.JsonStringHashMap;
@@ -60,8 +59,6 @@ import org.joda.time.LocalDateTime;
import org.junit.Assert;
import org.junit.Test;
-import io.netty.buffer.ArrowBuf;
-
public class TestComplexWriter {
private static final BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
@@ -101,7 +98,7 @@ public class TestComplexWriter {
}
private MapVector populateMapVector(CallBack callBack) {
- MapVector parent = new MapVector("parent", allocator, callBack);
+ MapVector parent = new MapVector("parent", allocator, new FieldType(false, Struct.INSTANCE, null, null), callBack);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
IntWriter intWriter = rootWriter.integer("int");
@@ -118,7 +115,7 @@ public class TestComplexWriter {
@Test
public void nullableMap() {
- try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+ try (MapVector mapVector = MapVector.empty("parent", allocator)) {
ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
MapWriter rootWriter = writer.rootAsMap();
for (int i = 0; i < COUNT; i++) {
@@ -142,7 +139,7 @@ public class TestComplexWriter {
*/
@Test
public void nullableMap2() {
- try (MapVector mapVector = new MapVector("parent", allocator, null)) {
+ try (MapVector mapVector = MapVector.empty("parent", allocator)) {
ComplexWriter writer = new ComplexWriterImpl("root", mapVector);
MapWriter rootWriter = writer.rootAsMap();
MapWriter mapWriter = rootWriter.map("map");
@@ -181,7 +178,7 @@ public class TestComplexWriter {
@Test
public void testList() {
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
@@ -210,7 +207,7 @@ public class TestComplexWriter {
@Test
public void listScalarType() {
- ListVector listVector = new ListVector("list", allocator, null, null);
+ ListVector listVector = ListVector.empty("list", allocator);
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
for (int i = 0; i < COUNT; i++) {
@@ -233,7 +230,7 @@ public class TestComplexWriter {
@Test
public void listScalarTypeNullable() {
- ListVector listVector = new ListVector("list", allocator, null, null);
+ ListVector listVector = ListVector.empty("list", allocator);
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
for (int i = 0; i < COUNT; i++) {
@@ -262,7 +259,7 @@ public class TestComplexWriter {
@Test
public void listMapType() {
- ListVector listVector = new ListVector("list", allocator, null, null);
+ ListVector listVector = ListVector.empty("list", allocator);
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
MapWriter mapWriter = listWriter.map();
@@ -290,7 +287,7 @@ public class TestComplexWriter {
@Test
public void listListType() {
- try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+ try (ListVector listVector = ListVector.empty("list", allocator)) {
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
for (int i = 0; i < COUNT; i++) {
@@ -315,7 +312,7 @@ public class TestComplexWriter {
*/
@Test
public void listListType2() {
- try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+ try (ListVector listVector = ListVector.empty("list", allocator)) {
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
ListWriter innerListWriter = listWriter.list();
@@ -353,7 +350,7 @@ public class TestComplexWriter {
@Test
public void unionListListType() {
- try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+ try (ListVector listVector = ListVector.empty("list", allocator)) {
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
for (int i = 0; i < COUNT; i++) {
@@ -382,7 +379,7 @@ public class TestComplexWriter {
*/
@Test
public void unionListListType2() {
- try (ListVector listVector = new ListVector("list", allocator, null, null)) {
+ try (ListVector listVector = ListVector.empty("list", allocator)) {
listVector.allocateNew();
UnionListWriter listWriter = new UnionListWriter(listVector);
ListWriter innerListWriter = listWriter.list();
@@ -454,7 +451,7 @@ public class TestComplexWriter {
@Test
public void promotableWriter() {
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
for (int i = 0; i < 100; i++) {
@@ -503,7 +500,7 @@ public class TestComplexWriter {
*/
@Test
public void promotableWriterSchema() {
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
rootWriter.bigInt("a");
@@ -536,7 +533,7 @@ public class TestComplexWriter {
@Test
public void mapWriterMixedCaseFieldNames() {
// test case-sensitive MapWriter
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("rootCaseSensitive", parent, false, true);
MapWriter rootWriterCaseSensitive = writer.rootAsMap();
rootWriterCaseSensitive.bigInt("int_field");
@@ -607,7 +604,7 @@ public class TestComplexWriter {
final LocalDateTime expectedNanoDateTime = expectedMilliDateTime;
// write
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
@@ -678,7 +675,7 @@ public class TestComplexWriter {
@Test
public void complexCopierWithList() {
- MapVector parent = new MapVector("parent", allocator, null);
+ MapVector parent = MapVector.empty("parent", allocator);
ComplexWriter writer = new ComplexWriterImpl("root", parent);
MapWriter rootWriter = writer.rootAsMap();
ListWriter listWriter = rootWriter.list("list");
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
index 3bed453..9156110 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFile.java
@@ -27,10 +27,13 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Map;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import org.apache.arrow.memory.BufferAllocator;
@@ -55,7 +58,9 @@ import org.apache.arrow.vector.schema.ArrowRecordBatch;
import org.apache.arrow.vector.stream.ArrowStreamReader;
import org.apache.arrow.vector.stream.ArrowStreamWriter;
import org.apache.arrow.vector.stream.MessageSerializerTest;
+import org.apache.arrow.vector.types.FloatingPointPrecision;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.FixedSizeList;
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
@@ -77,7 +82,7 @@ public class TestArrowFile extends BaseFileTest {
int count = COUNT;
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", vectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", vectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), file, new ByteArrayOutputStream());
}
@@ -89,7 +94,7 @@ public class TestArrowFile extends BaseFileTest {
int count = COUNT;
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+ NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
writeComplexData(count, parent);
FieldVector root = parent.getChild("root");
validateComplexContent(count, new VectorSchemaRoot(root));
@@ -105,7 +110,7 @@ public class TestArrowFile extends BaseFileTest {
// write
try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
writeData(count, parent);
write(parent.getChild("root"), file, stream);
}
@@ -172,7 +177,7 @@ public class TestArrowFile extends BaseFileTest {
// write
try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
writeComplexData(count, parent);
write(parent.getChild("root"), file, stream);
}
@@ -213,7 +218,7 @@ public class TestArrowFile extends BaseFileTest {
// write
try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorAllocator, null);
+ MapVector parent = MapVector.empty("parent", originalVectorAllocator);
FileOutputStream fileOutputStream = new FileOutputStream(file)){
writeData(counts[0], parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
@@ -286,7 +291,7 @@ public class TestArrowFile extends BaseFileTest {
// write
try (BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+ NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
writeUnionData(count, parent);
validateUnionData(count, new VectorSchemaRoot(parent.getChild("root")));
write(parent.getChild("root"), file, stream);
@@ -381,6 +386,77 @@ public class TestArrowFile extends BaseFileTest {
}
@Test
+ public void testWriteReadFieldMetadata() throws IOException {
+ File file = new File("target/mytest_metadata.arrow");
+ ByteArrayOutputStream stream = new ByteArrayOutputStream();
+
+ List<Field> childFields = new ArrayList<Field>();
+ childFields.add(new Field("varchar-child", new FieldType(true, ArrowType.Utf8.INSTANCE, null, metadata(1)), null));
+ childFields.add(new Field("float-child", new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null, metadata(2)), null));
+ childFields.add(new Field("int-child", new FieldType(false, new ArrowType.Int(32, true), null, metadata(3)), null));
+ childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)),
+ ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16 ,true)), null))));
+ Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields);
+ List<Field> fields = ImmutableList.of(field);
+
+ // write
+ try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
+ NullableMapVector vector = (NullableMapVector) field.createVector(originalVectorAllocator)) {
+ vector.allocateNewSafe();
+ vector.getMutator().setValueCount(0);
+
+ List<FieldVector> vectors = ImmutableList.<FieldVector>of(vector);
+ VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, 0);
+
+ try (FileOutputStream fileOutputStream = new FileOutputStream(file);
+ ArrowFileWriter fileWriter = new ArrowFileWriter(root, null, fileOutputStream.getChannel());
+ ArrowStreamWriter streamWriter = new ArrowStreamWriter(root, null, stream)) {
+ LOGGER.debug("writing schema: " + root.getSchema());
+ fileWriter.start();
+ streamWriter.start();
+ fileWriter.writeBatch();
+ streamWriter.writeBatch();
+ fileWriter.end();
+ streamWriter.end();
+ }
+ }
+
+ // read from file
+ try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+ FileInputStream fileInputStream = new FileInputStream(file);
+ ArrowFileReader arrowReader = new ArrowFileReader(fileInputStream.getChannel(), readerAllocator)) {
+ VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+ Schema schema = root.getSchema();
+ LOGGER.debug("reading schema: " + schema);
+ Assert.assertEquals(fields, schema.getFields());
+ Field top = schema.getFields().get(0);
+ Assert.assertEquals(metadata(0), top.getMetadata());
+ for (int i = 0; i < 4; i ++) {
+ Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
+ }
+ }
+
+ // Read from stream
+ try (BufferAllocator readerAllocator = allocator.newChildAllocator("reader", 0, Integer.MAX_VALUE);
+ ByteArrayInputStream input = new ByteArrayInputStream(stream.toByteArray());
+ ArrowStreamReader arrowReader = new ArrowStreamReader(input, readerAllocator)) {
+ VectorSchemaRoot root = arrowReader.getVectorSchemaRoot();
+ Schema schema = root.getSchema();
+ LOGGER.debug("reading schema: " + schema);
+ Assert.assertEquals(fields, schema.getFields());
+ Field top = schema.getFields().get(0);
+ Assert.assertEquals(metadata(0), top.getMetadata());
+ for (int i = 0; i < 4; i ++) {
+ Assert.assertEquals(metadata(i + 1), top.getChildren().get(i).getMetadata());
+ }
+ }
+ }
+
+ private Map<String, String> metadata(int i) {
+ return ImmutableMap.of("k_" + i, "v_" + i, "k2_" + i, "v2_" + i);
+ }
+
+ @Test
public void testWriteReadDictionary() throws IOException {
File file = new File("target/mytest_dict.arrow");
ByteArrayOutputStream stream = new ByteArrayOutputStream();
@@ -491,7 +567,7 @@ public class TestArrowFile extends BaseFileTest {
// write
try (NullableVarCharVector dictionaryVector = newNullableVarCharVector("dictionary", allocator);
- ListVector listVector = new ListVector("list", allocator, null, null)) {
+ ListVector listVector = ListVector.empty("list", allocator)) {
Dictionary dictionary = new Dictionary(dictionaryVector, encoding);
MapDictionaryProvider provider = new MapDictionaryProvider();
@@ -590,10 +666,10 @@ public class TestArrowFile extends BaseFileTest {
// write
try (BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", originalVectorAllocator, null, null)) {
- FixedSizeListVector tuples = parent.addOrGet("float-pairs", new FieldType(true, new FixedSizeList(2), null), FixedSizeListVector.class);
- NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(new FieldType(true, MinorType.FLOAT4.getType(), null)).getVector();
- NullableIntVector ints = parent.addOrGet("ints", new FieldType(true, new Int(32, true), null), NullableIntVector.class);
+ NullableMapVector parent = NullableMapVector.empty("parent", originalVectorAllocator)) {
+ FixedSizeListVector tuples = parent.addOrGet("float-pairs", FieldType.nullable(new FixedSizeList(2)), FixedSizeListVector.class);
+ NullableFloat4Vector floats = (NullableFloat4Vector) tuples.addOrGetVector(FieldType.nullable(MinorType.FLOAT4.getType())).getVector();
+ NullableIntVector ints = parent.addOrGet("ints", FieldType.nullable(new Int(32, true)), NullableIntVector.class);
parent.allocateNew();
for (int i = 0; i < 10; i++) {
@@ -641,6 +717,7 @@ public class TestArrowFile extends BaseFileTest {
}
}
+
/**
* Writes the contents of parents to file. If outStream is non-null, also writes it
* to outStream in the streaming serialized format.
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
index 1e51458..3014e64 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowFooter.java
@@ -28,6 +28,7 @@ import java.util.List;
import org.apache.arrow.flatbuf.Footer;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Test;
@@ -38,7 +39,7 @@ public class TestArrowFooter {
@Test
public void test() {
Schema schema = new Schema(asList(
- new Field("a", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())
+ new Field("a", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())
));
ArrowFooter footer = new ArrowFooter(schema, Collections.<ArrowBlock>emptyList(), Collections.<ArrowBlock>emptyList());
ArrowFooter newFooter = roundTrip(footer);
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
index d00cb0f..55629d5 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/TestArrowReaderWriter.java
@@ -41,6 +41,7 @@ import org.apache.arrow.vector.schema.ArrowFieldNode;
import org.apache.arrow.vector.schema.ArrowRecordBatch;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.arrow.vector.util.ByteArrayReadableSeekableByteChannel;
import org.junit.Before;
@@ -71,7 +72,7 @@ public class TestArrowReaderWriter {
@Test
public void test() throws IOException {
- Schema schema = new Schema(asList(new Field("testField", true, new ArrowType.Int(8, true), Collections.<Field>emptyList())));
+ Schema schema = new Schema(asList(new Field("testField", FieldType.nullable(new ArrowType.Int(8, true)), Collections.<Field>emptyList())));
ArrowType type = schema.getFields().get(0).getType();
FieldVector vector = TestUtils.newVector(FieldVector.class, "testField", type, allocator);
vector.initializeChildrenFromFields(schema.getFields().get(0).getChildren());
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
index 6369c07..6c29cab 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/json/TestJSONFile.java
@@ -43,7 +43,7 @@ public class TestJSONFile extends BaseFileTest {
// write
try (
BufferAllocator originalVectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- MapVector parent = new MapVector("parent", originalVectorAllocator, null)) {
+ MapVector parent = MapVector.empty("parent", originalVectorAllocator)) {
writeComplexData(count, parent);
writeJSON(file, new VectorSchemaRoot(parent.getChild("root")));
}
@@ -70,7 +70,7 @@ public class TestJSONFile extends BaseFileTest {
int count = COUNT;
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+ NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
writeComplexData(count, parent);
VectorSchemaRoot root = new VectorSchemaRoot(parent.getChild("root"));
validateComplexContent(root.getRowCount(), root);
@@ -92,7 +92,7 @@ public class TestJSONFile extends BaseFileTest {
int count = COUNT;
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+ NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
writeUnionData(count, parent);
@@ -127,7 +127,7 @@ public class TestJSONFile extends BaseFileTest {
// write
try (
BufferAllocator vectorAllocator = allocator.newChildAllocator("original vectors", 0, Integer.MAX_VALUE);
- NullableMapVector parent = new NullableMapVector("parent", vectorAllocator, null, null)) {
+ NullableMapVector parent = NullableMapVector.empty("parent", vectorAllocator)) {
writeDateTimeData(count, parent);
http://git-wip-us.apache.org/repos/asf/arrow/blob/010bd224/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index f9c8f72..64f7970 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -32,6 +32,7 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Test;
@@ -45,25 +46,25 @@ public class TestConvert {
@Test
public void simple() {
- Field initialField = new Field("a", true, new Int(32, true), null);
+ Field initialField = new Field("a", FieldType.nullable(new Int(32, true)), null);
run(initialField);
}
@Test
public void complex() {
ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
- childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
+ childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
- Field initialField = new Field("a", true, Struct.INSTANCE, childrenBuilder.build());
+ Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build());
run(initialField);
}
@Test
public void schema() {
ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
- childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
+ childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
Schema initialSchema = new Schema(childrenBuilder.build());
run(initialSchema);
}
@@ -71,18 +72,18 @@ public class TestConvert {
@Test
public void nestedSchema() {
ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", true, Utf8.INSTANCE, null));
- childrenBuilder.add(new Field("child2", true, new FloatingPoint(SINGLE), ImmutableList.<Field>of()));
- childrenBuilder.add(new Field("child3", true, new Struct(), ImmutableList.<Field>of(
- new Field("child3.1", true, Utf8.INSTANCE, null),
- new Field("child3.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+ childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
+ childrenBuilder.add(new Field("child3", FieldType.nullable(new Struct()), ImmutableList.<Field>of(
+ new Field("child3.1", FieldType.nullable(Utf8.INSTANCE), null),
+ new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
)));
- childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
- new Field("child4.1", true, Utf8.INSTANCE, null)
+ childrenBuilder.add(new Field("child4", FieldType.nullable(new List()), ImmutableList.<Field>of(
+ new Field("child4.1", FieldType.nullable(Utf8.INSTANCE), null)
)));
- childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
- new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND, null), null),
- new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
+ childrenBuilder.add(new Field("child5", FieldType.nullable(new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } )), ImmutableList.<Field>of(
+ new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null),
+ new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
)));
Schema initialSchema = new Schema(childrenBuilder.build());
run(initialSchema);