You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2018/09/25 23:04:23 UTC
[arrow] branch master updated: ARROW-249: [JAVA] Flight GRPC
Implementation
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new b3cd616 ARROW-249: [JAVA] Flight GRPC Implementation
b3cd616 is described below
commit b3cd616f6ce39e9a820b658ba2f2e2fd3b153555
Author: Jacques Nadeau <ja...@apache.org>
AuthorDate: Tue Sep 25 19:04:12 2018 -0400
ARROW-249: [JAVA] Flight GRPC Implementation
- A grpc defined protocol (flight.proto)
- A Java implementation of the GRPC-based FlightService framework
- An Example Java implementation of a FlightService that provides an in-memory store for Flight streams
- A short demo script to show how to use the FlightService from Java and Python
Author: Jacques Nadeau <ja...@apache.org>
Closes #2102 from jacques-n/flight and squashes the following commits:
135a2d789 <Jacques Nadeau> Update NOTICE for additional source code included.
01bdc0765 <Jacques Nadeau> Update for new checkstyle rules
2d00e1f24 <Jacques Nadeau> Fix checkstyle warnings
c5a065a0e <Jacques Nadeau> Update code for change in FlightService GRPC package.
723914d55 <Jacques Nadeau> Ensure full shutdown of flight server.
12924a26a <Jacques Nadeau> Add license headers
1e5733f11 <Jacques Nadeau> Attempt to correct memory leakage on Linux. Enable assertions in flight temporarily to see where memory is leaking.
fc9a49093 <Jacques Nadeau> Remove use of Guava from arrow-vector and arrow-memory modules so change in version doesn't impact downstream consumers.
03babacfe <Jacques Nadeau> Rebase and address new checkstyle rules.
0a12b358e <Jacques Nadeau> More Updates
d42d58f21 <Jacques Nadeau> backpressure explore
efd6409f4 <Jacques Nadeau> Update action to support streaming response. Add initial proto for handshake.
faf567f6a <Jacques Nadeau> Initial perf test server
7f1f2d100 <Jacques Nadeau> More cleanup and comments on proto.
b7e45f8d3 <Jacques Nadeau> More comments and some cleanups based on comments
d5be1fbea <Jacques Nadeau> ARROW-249 Flight GRPC Implementation
---
.gitignore | 1 +
NOTICE.txt | 8 +
java/flight/README.md | 95 ++
java/flight/pom.xml | 235 ++++
.../main/java/org/apache/arrow/flight/Action.java} | 49 +-
.../java/org/apache/arrow/flight/ActionType.java} | 40 +-
.../java/org/apache/arrow/flight/ArrowMessage.java | 333 +++++
.../java/org/apache/arrow/flight/Criteria.java} | 38 +-
.../apache/arrow/flight/FlightBindingService.java | 130 ++
.../java/org/apache/arrow/flight/FlightClient.java | 258 ++++
.../org/apache/arrow/flight/FlightConstants.java} | 26 +-
.../org/apache/arrow/flight/FlightDescriptor.java | 151 +++
.../org/apache/arrow/flight/FlightEndpoint.java | 64 +
.../java/org/apache/arrow/flight/FlightInfo.java | 87 ++
.../org/apache/arrow/flight/FlightProducer.java | 70 ++
.../java/org/apache/arrow/flight/FlightServer.java | 101 ++
.../org/apache/arrow/flight/FlightService.java | 182 +++
.../java/org/apache/arrow/flight/FlightStream.java | 210 ++++
.../org/apache/arrow/flight/GenericOperation.java} | 31 +-
.../java/org/apache/arrow/flight/Location.java} | 42 +-
.../apache/arrow/flight/NoOpFlightProducer.java | 57 +
.../main/java/org/apache/arrow/flight/Result.java} | 37 +-
.../java/org/apache/arrow/flight/StreamPipe.java | 62 +
.../main/java/org/apache/arrow/flight/Ticket.java | 79 ++
.../apache/arrow/flight/auth/AuthConstants.java} | 50 +-
.../arrow/flight/auth/BasicClientAuthHandler.java} | 43 +-
.../arrow/flight/auth/BasicServerAuthHandler.java | 72 ++
.../arrow/flight/auth/ClientAuthHandler.java} | 30 +-
.../arrow/flight/auth/ClientAuthInterceptor.java | 72 ++
.../arrow/flight/auth/ClientAuthWrapper.java | 125 ++
.../arrow/flight/auth/ServerAuthHandler.java} | 45 +-
.../arrow/flight/auth/ServerAuthInterceptor.java | 57 +
.../arrow/flight/auth/ServerAuthWrapper.java | 136 ++
.../arrow/flight/example/ExampleFlightServer.java | 82 ++
.../apache/arrow/flight/example/ExampleTicket.java | 126 ++
.../apache/arrow/flight/example/FlightHolder.java | 89 ++
.../apache/arrow/flight/example/InMemoryStore.java | 157 +++
.../org/apache/arrow/flight/example/Stream.java | 130 ++
.../arrow/flight/grpc/AddWritableBuffer.java | 121 ++
.../arrow/flight/grpc/GetReadableBuffer.java | 70 ++
.../org/apache/arrow/flight/TestBackPressure.java | 162 +++
.../apache/arrow/flight/TestBasicOperation.java | 257 ++++
.../org/apache/arrow/flight/auth/TestAuth.java | 120 ++
.../arrow/flight/example/TestExampleServer.java | 110 ++
.../arrow/flight/perf/PerformanceTestServer.java | 203 +++
.../org/apache/arrow/flight/perf/TestPerf.java | 184 +++
.../src/test/protobuf/perf.proto} | 44 +-
java/flight/src/test/resources/logback.xml | 27 +
java/memory/pom.xml | 4 -
.../src/main/java/io/netty/buffer/ArrowBuf.java | 3 +-
.../java/org/apache/arrow/memory/Accountant.java | 2 +-
.../org/apache/arrow/memory/AllocationManager.java | 5 +-
.../org/apache/arrow/memory/BaseAllocator.java | 3 +-
.../arrow/memory/LowCostIdentityHashMap.java | 4 +-
.../org/apache/arrow/memory/RootAllocator.java | 2 +-
.../java/org/apache/arrow/util/Preconditions.java | 1306 ++++++++++++++++++++
.../org/apache/arrow/util/VisibleForTesting.java} | 25 +-
java/pom.xml | 6 +-
java/vector/pom.xml | 8 -
.../src/main/codegen/includes/vv_imports.ftl | 11 +-
.../src/main/codegen/templates/StructWriters.java | 5 +-
.../src/main/codegen/templates/UnionVector.java | 10 +-
.../src/main/codegen/templates/UnionWriter.java | 2 +-
.../java/org/apache/arrow/util/AutoCloseables.java | 205 +++
.../java/org/apache/arrow/util/Collections2.java | 54 +
.../org/apache/arrow/vector/AddOrGetResult.java | 2 +-
.../org/apache/arrow/vector/BaseValueVector.java | 3 +-
.../java/org/apache/arrow/vector/BufferLayout.java | 2 +-
.../java/org/apache/arrow/vector/TypeLayout.java | 3 +-
.../java/org/apache/arrow/vector/VectorLoader.java | 7 +-
.../org/apache/arrow/vector/VectorSchemaRoot.java | 70 +-
.../arrow/vector/complex/AbstractStructVector.java | 18 +-
.../vector/complex/BaseRepeatedValueVector.java | 13 +-
.../arrow/vector/complex/EmptyValuePopulator.java | 3 +-
.../arrow/vector/complex/FixedSizeListVector.java | 34 +-
.../apache/arrow/vector/complex/ListVector.java | 17 +-
.../vector/complex/NonNullableStructVector.java | 22 +-
.../apache/arrow/vector/complex/StructVector.java | 17 +-
.../vector/complex/impl/ComplexWriterImpl.java | 3 +-
.../complex/impl/SingleStructReaderImpl.java | 5 +-
.../arrow/vector/dictionary/DictionaryEncoder.java | 5 +-
.../apache/arrow/vector/ipc/ArrowFileWriter.java | 3 +-
.../org/apache/arrow/vector/ipc/ArrowReader.java | 6 +-
.../org/apache/arrow/vector/ipc/ArrowWriter.java | 7 +-
.../apache/arrow/vector/ipc/JsonFileReader.java | 35 +-
.../apache/arrow/vector/ipc/JsonFileWriter.java | 38 +-
.../arrow/vector/ipc/message/ArrowRecordBatch.java | 34 +
.../org/apache/arrow/vector/types/pojo/Field.java | 28 +-
.../apache/arrow/vector/types/pojo/FieldType.java | 10 +-
.../org/apache/arrow/vector/types/pojo/Schema.java | 38 +-
.../arrow/vector/util/ByteFunctionHelpers.java | 18 +-
.../apache/arrow/vector/util/MapWithOrdinal.java | 33 +-
.../org/apache/arrow/vector/util/Validator.java | 5 +-
.../arrow/vector/TestFixedSizeListVector.java | 14 +-
.../org/apache/arrow/vector/ipc/BaseFileTest.java | 15 +-
.../org/apache/arrow/vector/ipc/TestArrowFile.java | 24 +-
.../org/apache/arrow/vector/pojo/TestConvert.java | 62 +-
97 files changed, 6631 insertions(+), 516 deletions(-)
diff --git a/.gitignore b/.gitignore
index 1493582..e54be5f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,7 @@
*.so.*
*.dylib
.build_cache_dir
+dependency-reduced-pom.xml
MANIFEST
# Generated Visual Studio files
diff --git a/NOTICE.txt b/NOTICE.txt
index 875ad74..b4664a5 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -41,6 +41,14 @@ This product includes software from the Ibis project (Apache 2.0)
* Copyright (c) 2015 Cloudera, Inc.
* https://github.com/cloudera/ibis
+This product includes software from Dremio (Apache 2.0)
+ * Copyright (C) 2017-2018 Dremio Corporation
+ * https://github.com/dremio/dremio-oss
+
+This product includes software from Google Guava (Apache 2.0)
+ * Copyright (C) 2007 The Guava Authors
+ * https://github.com/google/guava
+
The web site includes files generated by Jekyll.
--------------------------------------------------------------------------------
diff --git a/java/flight/README.md b/java/flight/README.md
new file mode 100644
index 0000000..da91a4f
--- /dev/null
+++ b/java/flight/README.md
@@ -0,0 +1,95 @@
+<!---
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Arrow Flight Java Package
+
+Exposing Apache Arrow data on the wire.
+
+[Protocol Description Slides](https://www.slideshare.net/JacquesNadeau5/apache-arrow-flight-overview)
+
+[GRPC Protocol Definition](https://github.com/jacques-n/arrow/blob/flight/java/flight/src/main/protobuf/flight.proto)
+
+## Example usage
+
+* Compile the java tree:
+
+ ```
+ cd java
+ mvn clean install -DskipTests
+ ```
+
+* Go Into the Flight tree
+
+ ```
+ cd flight
+ ```
+
+
+* Start the ExampleFlightServer (supports get/put of streams and listing these streams)
+
+ ```
+ mvn exec:exec
+ ```
+
+* In new terminal, run the TestExampleServer to populate the server with example data
+
+ ```
+ cd arrow/java/flight
+ mvn surefire:test -DdisableServer=true -Dtest=TestExampleServer
+ ```
+
+## Python Example Usage
+
+* Compile example python headers
+
+ ```
+ mkdir target/generated-python
+ pip install grpcio-tools # or conda install grpcio
+ python -m grpc_tools.protoc -I./src/main/protobuf/ --python_out=./target/generated-python --grpc_python_out=./target/generated-python src/main/protobuf/flight.proto
+ ```
+
+* Connect to the Flight Service
+
+ ```
+ cd target/generated-python
+ python
+ ```
+
+
+ ```
+ import grpc
+ import flight_pb2
+ import flight_pb2_grpc as flightrpc
+ channel = grpc.insecure_channel('localhost:12233')
+ service = flightrpc.FlightServiceStub(channel)
+ ```
+
+* List the Flight from Python
+
+ ```
+ for f in service.ListFlights(flight_pb2.Criteria()): f
+ ```
+
+* Try to Drop
+
+ ```
+ action = flight_pb2.Action()
+ action.type="drop"
+ service.DoAction(action)
+ ```
diff --git a/java/flight/pom.xml b/java/flight/pom.xml
new file mode 100644
index 0000000..642fd12
--- /dev/null
+++ b/java/flight/pom.xml
@@ -0,0 +1,235 @@
+<?xml version="1.0"?>
+<!-- Copyright (C) 2017-2018 Dremio Corporation Licensed under the Apache
+ License, Version 2.0 (the "License"); you may not use this file except in
+ compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License. -->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-java-root</artifactId>
+ <version>0.11.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>arrow-flight</artifactId>
+ <name>Arrow Flight</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <dep.grpc.version>1.14.0</dep.grpc.version>
+ <dep.protobuf.version>3.5.1</dep.protobuf.version>
+ <forkCount>1</forkCount>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.vlkan</groupId>
+ <artifactId>flatbuffers</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-vector</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-format</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty</artifactId>
+ <version>${dep.grpc.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-netty</artifactId>
+ <version>${dep.grpc.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-core</artifactId>
+ <version>${dep.grpc.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-protobuf</artifactId>
+ <version>${dep.grpc.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ <version>2.0.12.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.grpc</groupId>
+ <artifactId>grpc-stub</artifactId>
+ <version>${dep.grpc.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>3.5.1</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <extensions>
+ <extension>
+ <groupId>kr.motd.maven</groupId>
+ <artifactId>os-maven-plugin</artifactId>
+ <version>1.5.0.Final</version>
+ </extension>
+ </extensions>
+ <plugins>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <enableAssertions>false</enableAssertions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>3.1.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <shadedArtifactAttached>true</shadedArtifactAttached>
+ <shadedClassifierName>shaded</shadedClassifierName>
+ <artifactSet>
+ <includes>
+ <include>io.grpc:*</include>
+ <include>com.google.protobuf:*</include>
+ </includes>
+ </artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.protobuf</pattern>
+ <shadedPattern>arrow.flight.com.google.protobuf</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.xolstice.maven.plugins</groupId>
+ <artifactId>protobuf-maven-plugin</artifactId>
+ <version>0.5.0</version>
+ <configuration>
+ <protocArtifact>com.google.protobuf:protoc:${dep.protobuf.version}:exe:${os.detected.classifier}</protocArtifact>
+ <clearOutputDirectory>false</clearOutputDirectory>
+ <pluginId>grpc-java</pluginId>
+ <pluginArtifact>io.grpc:protoc-gen-grpc-java:${dep.grpc.version}:exe:${os.detected.classifier}</pluginArtifact>
+ </configuration>
+ <executions>
+ <execution>
+ <id>src</id>
+ <configuration>
+ <protoSourceRoot>${basedir}/../../format/</protoSourceRoot>
+ <outputDirectory>${project.build.directory}/generated-sources/protobuf</outputDirectory>
+ </configuration>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>test</id>
+ <configuration>
+ <protoSourceRoot>${basedir}/src/test/protobuf</protoSourceRoot>
+ <outputDirectory>${project.build.directory}/generated-test-sources//protobuf</outputDirectory>
+ </configuration>
+ <goals>
+ <goal>compile</goal>
+ <goal>compile-custom</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.6.0</version>
+ <configuration>
+ <executable>java</executable>
+ <classpathScope>test</classpathScope>
+ <arguments>
+ <argument>-classpath</argument>
+ <classpath/>
+ <argument>-Xms64m</argument>
+ <argument>-Xmx64m</argument>
+ <argument>-XX:MaxDirectMemorySize=4g</argument>
+ <argument>org.apache.arrow.flight.example.ExampleFlightServer</argument>
+ </arguments>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>analyze</id>
+ <phase>verify</phase>
+ <goals>
+ <goal>analyze-only</goal>
+ </goals>
+ <configuration>
+ <ignoredDependencies combine.self="append">
+ <ignoredDependency>io.netty:netty-tcnative-boringssl-static:*</ignoredDependency>
+ </ignoredDependencies>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/flight/src/main/java/org/apache/arrow/flight/Action.java
similarity index 52%
copy from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
copy to java/flight/src/main/java/org/apache/arrow/flight/Action.java
index 161b81a..0035b87 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/Action.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,29 +16,42 @@
* limitations under the License.
*/
-package org.apache.arrow.memory;
+package org.apache.arrow.flight;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.arrow.flight.impl.Flight;
-/**
- * The root allocator for using direct memory inside a Drillbit. Supports creating a
- * tree of descendant child allocators.
- */
-public class RootAllocator extends BaseAllocator {
+import com.google.protobuf.ByteString;
+
+public class Action {
+
+ private final String type;
+ private final byte[] body;
+
+ public Action(String type) {
+ this(type, null);
+ }
+
+ public Action(String type, byte[] body) {
+ this.type = type;
+ this.body = body == null ? new byte[0] : body;
+ }
+
+ public Action(Flight.Action action) {
+ this(action.getType(), action.getBody().toByteArray());
+ }
- public RootAllocator(final long limit) {
- this(AllocationListener.NOOP, limit);
+ public String getType() {
+ return type;
}
- public RootAllocator(final AllocationListener listener, final long limit) {
- super(null, listener, "ROOT", 0, limit);
+ public byte[] getBody() {
+ return body;
}
- /**
- * Verify the accounting state of the allocation system.
- */
- @VisibleForTesting
- public void verify() {
- verifyAllocator();
+ Flight.Action toProtocol() {
+ return Flight.Action.newBuilder()
+ .setType(getType())
+ .setBody(ByteString.copyFrom(getBody()))
+ .build();
}
}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/flight/src/main/java/org/apache/arrow/flight/ActionType.java
similarity index 56%
copy from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
copy to java/flight/src/main/java/org/apache/arrow/flight/ActionType.java
index 161b81a..bcdf17e 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/ActionType.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,29 +16,33 @@
* limitations under the License.
*/
-package org.apache.arrow.memory;
+package org.apache.arrow.flight;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.arrow.flight.impl.Flight;
-/**
- * The root allocator for using direct memory inside a Drillbit. Supports creating a
- * tree of descendant child allocators.
- */
-public class RootAllocator extends BaseAllocator {
+public class ActionType {
+ private final String type;
+ private final String description;
+
+ public ActionType(String type, String description) {
+ super();
+ this.type = type;
+ this.description = description;
+ }
- public RootAllocator(final long limit) {
- this(AllocationListener.NOOP, limit);
+ ActionType(Flight.ActionType type) {
+ this.type = type.getType();
+ this.description = type.getDescription();
}
- public RootAllocator(final AllocationListener listener, final long limit) {
- super(null, listener, "ROOT", 0, limit);
+ public String getType() {
+ return type;
}
- /**
- * Verify the accounting state of the allocation system.
- */
- @VisibleForTesting
- public void verify() {
- verifyAllocator();
+ Flight.ActionType toProtocol() {
+ return Flight.ActionType.newBuilder()
+ .setType(type)
+ .setDescription(description)
+ .build();
}
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/ArrowMessage.java b/java/flight/src/main/java/org/apache/arrow/flight/ArrowMessage.java
new file mode 100644
index 0000000..230629f
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/ArrowMessage.java
@@ -0,0 +1,333 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.arrow.flatbuf.Message;
+import org.apache.arrow.flatbuf.MessageHeader;
+import org.apache.arrow.flatbuf.RecordBatch;
+import org.apache.arrow.flight.grpc.AddWritableBuffer;
+import org.apache.arrow.flight.grpc.GetReadableBuffer;
+import org.apache.arrow.flight.impl.Flight.FlightData;
+import org.apache.arrow.flight.impl.Flight.FlightDescriptor;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.ipc.message.MessageSerializer;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.io.ByteStreams;
+import com.google.flatbuffers.FlatBufferBuilder;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.CodedOutputStream;
+import com.google.protobuf.WireFormat;
+
+import io.grpc.Drainable;
+import io.grpc.MethodDescriptor;
+import io.grpc.MethodDescriptor.Marshaller;
+import io.grpc.internal.ReadableBuffer;
+import io.grpc.protobuf.ProtoUtils;
+import io.netty.buffer.ArrowBuf;
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufInputStream;
+import io.netty.buffer.CompositeByteBuf;
+
+/**
+ * The in-memory representation of FlightData used to manage a stream of Arrow messages.
+ */
+class ArrowMessage implements AutoCloseable {
+
+ public static final boolean FAST_PATH = true;
+
+ private static final int DESCRIPTOR_TAG =
+ (FlightData.FLIGHT_DESCRIPTOR_FIELD_NUMBER << 3) | WireFormat.WIRETYPE_LENGTH_DELIMITED;
+ private static final int BODY_TAG =
+ (FlightData.DATA_BODY_FIELD_NUMBER << 3) | WireFormat.WIRETYPE_LENGTH_DELIMITED;
+ private static final int HEADER_TAG =
+ (FlightData.DATA_HEADER_FIELD_NUMBER << 3) | WireFormat.WIRETYPE_LENGTH_DELIMITED;
+
+ private static Marshaller<FlightData> NO_BODY_MARSHALLER = ProtoUtils.marshaller(FlightData.getDefaultInstance());
+
+ public static enum HeaderType {
+ NONE,
+ SCHEMA,
+ DICTIONARY_BATCH,
+ RECORD_BATCH,
+ TENSOR
+ ;
+
+ public static HeaderType getHeader(byte b) {
+ switch (b) {
+ case 0: return NONE;
+ case 1: return SCHEMA;
+ case 2: return DICTIONARY_BATCH;
+ case 3: return RECORD_BATCH;
+ case 4: return TENSOR;
+ default:
+ throw new UnsupportedOperationException("unknown type: " + b);
+ }
+ }
+
+ }
+
+ private final FlightDescriptor descriptor;
+ private final Message message;
+ private final List<ArrowBuf> bufs;
+
+ public ArrowMessage(FlightDescriptor descriptor, Schema schema) {
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ int schemaOffset = schema.getSchema(builder);
+ ByteBuffer serializedMessage = MessageSerializer.serializeMessage(builder, MessageHeader.Schema, schemaOffset, 0);
+ serializedMessage = serializedMessage.slice();
+ message = Message.getRootAsMessage(serializedMessage);
+ bufs = ImmutableList.of();
+ this.descriptor = descriptor;
+ }
+
+ public ArrowMessage(ArrowRecordBatch batch) {
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ int batchOffset = batch.writeTo(builder);
+ ByteBuffer serializedMessage = MessageSerializer.serializeMessage(builder, MessageHeader.RecordBatch, batchOffset,
+ batch.computeBodyLength());
+ serializedMessage = serializedMessage.slice();
+ this.message = Message.getRootAsMessage(serializedMessage);
+ this.bufs = ImmutableList.copyOf(batch.getBuffers());
+ this.descriptor = null;
+ }
+
+ private ArrowMessage(FlightDescriptor descriptor, Message message, ArrowBuf buf) {
+ this.message = message;
+ this.descriptor = descriptor;
+ this.bufs = buf == null ? ImmutableList.of() : ImmutableList.of(buf);
+ }
+
+ public FlightDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ public HeaderType getMessageType() {
+ return HeaderType.getHeader(message.headerType());
+ }
+
+ public Message asSchemaMessage() {
+ return message;
+ }
+
+ public Schema asSchema() {
+ Preconditions.checkArgument(bufs.size() == 0);
+ Preconditions.checkArgument(getMessageType() == HeaderType.SCHEMA);
+ org.apache.arrow.flatbuf.Schema schema = new org.apache.arrow.flatbuf.Schema();
+ message.header(schema);
+ return Schema.convertSchema(schema);
+ }
+
+ public ArrowRecordBatch asRecordBatch() throws IOException {
+ Preconditions.checkArgument(bufs.size() == 1, "A batch can only be consumed if it contains a single ArrowBuf.");
+ Preconditions.checkArgument(getMessageType() == HeaderType.RECORD_BATCH);
+ RecordBatch recordBatch = new RecordBatch();
+ message.header(recordBatch);
+ ArrowBuf underlying = bufs.get(0);
+ ArrowRecordBatch batch = MessageSerializer.deserializeRecordBatch(recordBatch, underlying);
+ return batch;
+ }
+
+ public Iterable<ArrowBuf> getBufs() {
+ return Iterables.unmodifiableIterable(bufs);
+ }
+
+ private static ArrowMessage frame(BufferAllocator allocator, final InputStream stream) {
+
+ try {
+ FlightDescriptor descriptor = null;
+ Message header = null;
+ ArrowBuf body = null;
+ while (stream.available() > 0) {
+ int tag = readRawVarint32(stream);
+ switch (tag) {
+
+ case DESCRIPTOR_TAG: {
+ int size = readRawVarint32(stream);
+ byte[] bytes = new byte[size];
+ ByteStreams.readFully(stream, bytes);
+ descriptor = FlightDescriptor.parseFrom(bytes);
+ break;
+ }
+ case HEADER_TAG: {
+ int size = readRawVarint32(stream);
+ byte[] bytes = new byte[size];
+ ByteStreams.readFully(stream, bytes);
+ header = Message.getRootAsMessage(ByteBuffer.wrap(bytes));
+ break;
+ }
+ case BODY_TAG:
+ if (body != null) {
+ // only read last body.
+ body.release();
+ body = null;
+ }
+ int size = readRawVarint32(stream);
+ body = allocator.buffer(size);
+ ReadableBuffer readableBuffer = FAST_PATH ? GetReadableBuffer.getReadableBuffer(stream) : null;
+ if (readableBuffer != null) {
+ readableBuffer.readBytes(body.nioBuffer(0, size));
+ } else {
+ byte[] heapBytes = new byte[size];
+ ByteStreams.readFully(stream, heapBytes);
+ body.writeBytes(heapBytes);
+ }
+ body.writerIndex(size);
+ break;
+
+ default:
+ // ignore unknown fields.
+ }
+ }
+
+ return new ArrowMessage(descriptor, header, body);
+ } catch (Exception ioe) {
+ throw new RuntimeException(ioe);
+ }
+
+ }
+
+ private static int readRawVarint32(InputStream is) throws IOException {
+ int firstByte = is.read();
+ return CodedInputStream.readRawVarint32(firstByte, is);
+ }
+
+ /**
+ * Convert the ArrowMessage to an InputStream.
+ * @return
+ */
+ private InputStream asInputStream(BufferAllocator allocator) {
+ try {
+
+ final ByteString bytes = ByteString.copyFrom(message.getByteBuffer(), message.getByteBuffer().remaining());
+
+
+ if (getMessageType() == HeaderType.SCHEMA) {
+
+ final FlightData.Builder builder = FlightData.newBuilder()
+ .setDataHeader(bytes);
+
+ if (descriptor != null) {
+ builder.setFlightDescriptor(descriptor);
+ }
+
+ Preconditions.checkArgument(bufs.isEmpty());
+ return NO_BODY_MARSHALLER.stream(builder.build());
+ }
+
+ Preconditions.checkArgument(getMessageType() == HeaderType.RECORD_BATCH);
+ Preconditions.checkArgument(!bufs.isEmpty());
+ Preconditions.checkArgument(descriptor == null, "Descriptor should only be included in the schema message.");
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ CodedOutputStream cos = CodedOutputStream.newInstance(baos);
+ cos.writeBytes(FlightData.DATA_HEADER_FIELD_NUMBER, bytes);
+ cos.writeTag(FlightData.DATA_BODY_FIELD_NUMBER, WireFormat.WIRETYPE_LENGTH_DELIMITED);
+
+ int size = 0;
+ for (ArrowBuf b : bufs) {
+ size += b.readableBytes();
+ }
+ // rawvarint is used for length definition.
+ cos.writeUInt32NoTag(size);
+ cos.flush();
+
+ ArrowBuf initialBuf = allocator.buffer(baos.size());
+ initialBuf.writeBytes(baos.toByteArray());
+ final CompositeByteBuf bb = new CompositeByteBuf(allocator.getAsByteBufAllocator(), true, bufs.size() + 1,
+ ImmutableList.<ByteBuf>builder().add(initialBuf).addAll(bufs).build());
+ final ByteBufInputStream is = new DrainableByteBufInputStream(bb);
+ return is;
+ } catch (Exception ex) {
+ throw new RuntimeException("Unexpected IO Exception", ex);
+ }
+
+ }
+
+ private class DrainableByteBufInputStream extends ByteBufInputStream implements Drainable {
+
+ private final CompositeByteBuf buf;
+
+ public DrainableByteBufInputStream(CompositeByteBuf buffer) {
+ super(buffer, buffer.readableBytes(), true);
+ this.buf = buffer;
+ }
+
+ @Override
+ public int drainTo(OutputStream target) throws IOException {
+ int size = buf.readableBytes();
+ if (FAST_PATH && AddWritableBuffer.add(buf, target)) {
+ return size;
+ }
+
+ buf.getBytes(0, target, buf.readableBytes());
+ return size;
+ }
+
+ @Override
+ public void close() throws IOException {
+ buf.release();
+ }
+
+
+
+ }
+
+ public static Marshaller<ArrowMessage> createMarshaller(BufferAllocator allocator) {
+ return new ArrowMessageHolderMarshaller(allocator);
+ }
+
+ private static class ArrowMessageHolderMarshaller implements MethodDescriptor.Marshaller<ArrowMessage> {
+
+ private final BufferAllocator allocator;
+
+ public ArrowMessageHolderMarshaller(BufferAllocator allocator) {
+ this.allocator = allocator;
+ }
+
+ @Override
+ public InputStream stream(ArrowMessage value) {
+ return value.asInputStream(allocator);
+ }
+
+ @Override
+ public ArrowMessage parse(InputStream stream) {
+ return ArrowMessage.frame(allocator, stream);
+ }
+
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(bufs);
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/Criteria.java
similarity index 57%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/Criteria.java
index a5b5c9d..b5c313c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/Criteria.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,32 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.flight.impl.Flight;
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+import com.google.protobuf.ByteString;
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
+public class Criteria {
+
+ public static Criteria ALL = new Criteria((byte[]) null);
+
+ private final byte[] bytes;
+
+ public Criteria(byte[] bytes) {
+ this.bytes = bytes;
}
- public V getVector() {
- return vector;
+ Criteria(Flight.Criteria criteria) {
+ this.bytes = criteria.getExpression().toByteArray();
}
- public boolean isCreated() {
- return created;
+ Flight.Criteria asCriteria() {
+ Flight.Criteria.Builder b = Flight.Criteria.newBuilder();
+ if (bytes != null) {
+ b.setExpression(ByteString.copyFrom(bytes));
+ }
+
+ return b.build();
}
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightBindingService.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightBindingService.java
new file mode 100644
index 0000000..bd444ab
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightBindingService.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.Set;
+
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.memory.BufferAllocator;
+
+import com.google.common.collect.ImmutableSet;
+
+import io.grpc.BindableService;
+import io.grpc.MethodDescriptor;
+import io.grpc.ServerMethodDefinition;
+import io.grpc.ServerServiceDefinition;
+import io.grpc.protobuf.ProtoUtils;
+import io.grpc.stub.ServerCalls;
+import io.grpc.stub.ServerCalls.ClientStreamingMethod;
+import io.grpc.stub.ServerCalls.ServerStreamingMethod;
+import io.grpc.stub.StreamObserver;
+
+/**
+ * Extends the basic flight service to override some methods for more efficient implementations.
+ */
+class FlightBindingService implements BindableService {
+
+ private static final String DO_GET = MethodDescriptor.generateFullMethodName(FlightConstants.SERVICE, "DoGet");
+ private static final String DO_PUT = MethodDescriptor.generateFullMethodName(FlightConstants.SERVICE, "DoPut");
+ private static final Set<String> OVERRIDE_METHODS = ImmutableSet.of(DO_GET, DO_PUT);
+
+ private final FlightService delegate;
+ private final BufferAllocator allocator;
+
+ public FlightBindingService(BufferAllocator allocator, FlightProducer producer,
+ ServerAuthHandler authHandler) {
+ this.allocator = allocator;
+ this.delegate = new FlightService(allocator, producer, authHandler);
+ }
+
+ public static MethodDescriptor<Flight.Ticket, ArrowMessage> getDoGetDescriptor(BufferAllocator allocator) {
+ return MethodDescriptor.<Flight.Ticket, ArrowMessage>newBuilder()
+ .setType(io.grpc.MethodDescriptor.MethodType.SERVER_STREAMING)
+ .setFullMethodName(DO_GET)
+ .setSampledToLocalTracing(false)
+ .setRequestMarshaller(ProtoUtils.marshaller(Flight.Ticket.getDefaultInstance()))
+ .setResponseMarshaller(ArrowMessage.createMarshaller(allocator))
+ .build();
+ }
+
+ public static MethodDescriptor<ArrowMessage, Flight.PutResult> getDoPutDescriptor(BufferAllocator allocator) {
+ return MethodDescriptor.<ArrowMessage, Flight.PutResult>newBuilder()
+ .setType(io.grpc.MethodDescriptor.MethodType.CLIENT_STREAMING)
+ .setFullMethodName(DO_PUT)
+ .setSampledToLocalTracing(false)
+ .setRequestMarshaller(ArrowMessage.createMarshaller(allocator))
+ .setResponseMarshaller(ProtoUtils.marshaller(Flight.PutResult.getDefaultInstance()))
+ .build();
+ }
+
+ @Override
+ public ServerServiceDefinition bindService() {
+ final ServerServiceDefinition baseDefinition = delegate.bindService();
+
+ final MethodDescriptor<Flight.Ticket, ArrowMessage> doGetDescriptor = getDoGetDescriptor(allocator);
+
+ final MethodDescriptor<ArrowMessage, Flight.PutResult> doPutDescriptor = getDoPutDescriptor(allocator);
+
+ ServerServiceDefinition.Builder serviceBuilder = ServerServiceDefinition.builder(FlightConstants.SERVICE);
+ serviceBuilder.addMethod(doGetDescriptor, ServerCalls.asyncServerStreamingCall(new DoGetMethod(delegate)));
+ serviceBuilder.addMethod(doPutDescriptor, ServerCalls.asyncClientStreamingCall(new DoPutMethod(delegate)));
+
+ // copy over not-overridden methods.
+ for (ServerMethodDefinition<?, ?> definition : baseDefinition.getMethods()) {
+ if (OVERRIDE_METHODS.contains(definition.getMethodDescriptor().getFullMethodName())) {
+ continue;
+ }
+
+ serviceBuilder.addMethod(definition);
+ }
+
+ return serviceBuilder.build();
+ }
+
+ private class DoGetMethod implements ServerStreamingMethod<Flight.Ticket, ArrowMessage> {
+
+ private final FlightService delegate;
+
+ public DoGetMethod(FlightService delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void invoke(Flight.Ticket request, StreamObserver<ArrowMessage> responseObserver) {
+ delegate.doGetCustom(request, responseObserver);
+ }
+ }
+
+ private class DoPutMethod implements ClientStreamingMethod<ArrowMessage, Flight.PutResult> {
+ private final FlightService delegate;
+
+ public DoPutMethod(FlightService delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public StreamObserver<ArrowMessage> invoke(StreamObserver<PutResult> responseObserver) {
+ return delegate.doPutCustom(responseObserver);
+ }
+
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightClient.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightClient.java
new file mode 100644
index 0000000..f108e51
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightClient.java
@@ -0,0 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import static io.grpc.stub.ClientCalls.asyncClientStreamingCall;
+import static io.grpc.stub.ClientCalls.asyncServerStreamingCall;
+
+import java.util.Iterator;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.auth.BasicClientAuthHandler;
+import org.apache.arrow.flight.auth.ClientAuthHandler;
+import org.apache.arrow.flight.auth.ClientAuthInterceptor;
+import org.apache.arrow.flight.auth.ClientAuthWrapper;
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.Empty;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.flight.impl.FlightServiceGrpc;
+import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceBlockingStub;
+import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceStub;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+
+import io.grpc.ClientCall;
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.MethodDescriptor;
+import io.grpc.stub.ClientCallStreamObserver;
+import io.grpc.stub.ClientResponseObserver;
+import io.grpc.stub.StreamObserver;
+
+public class FlightClient implements AutoCloseable {
+ private final static int PENDING_REQUESTS = 5;
+ private final BufferAllocator allocator;
+ private final ManagedChannel channel;
+ private final FlightServiceBlockingStub blockingStub;
+ private final FlightServiceStub asyncStub;
+ private final ClientAuthInterceptor authInterceptor = new ClientAuthInterceptor();
+ private final MethodDescriptor<Flight.Ticket, ArrowMessage> doGetDescriptor;
+ private final MethodDescriptor<ArrowMessage, Flight.PutResult> doPutDescriptor;
+
+ /** Construct client for accessing RouteGuide server using the existing channel. */
+ public FlightClient(BufferAllocator incomingAllocator, Location location) {
+ final ManagedChannelBuilder<?> channelBuilder = ManagedChannelBuilder.forAddress(location.getHost(),
+ location.getPort()).maxTraceEvents(0).usePlaintext();
+ this.allocator = incomingAllocator.newChildAllocator("flight-client", 0, Long.MAX_VALUE);
+ channel = channelBuilder.build();
+ blockingStub = FlightServiceGrpc.newBlockingStub(channel).withInterceptors(authInterceptor);
+ asyncStub = FlightServiceGrpc.newStub(channel).withInterceptors(authInterceptor);
+ doGetDescriptor = FlightBindingService.getDoGetDescriptor(allocator);
+ doPutDescriptor = FlightBindingService.getDoPutDescriptor(allocator);
+ }
+
+ /**
+ * Get a list of available flights
+ * @param criteria
+ * @return
+ */
+ public Iterable<FlightInfo> listFlights(Criteria criteria) {
+ return ImmutableList.copyOf(blockingStub.listFlights(criteria.asCriteria()))
+ .stream()
+ .map(t -> new FlightInfo(t))
+ .collect(Collectors.toList());
+ }
+
+ public Iterable<ActionType> listActions() {
+ return ImmutableList.copyOf(blockingStub.listActions(Empty.getDefaultInstance()))
+ .stream()
+ .map(t -> new ActionType(t))
+ .collect(Collectors.toList());
+ }
+
+ public Iterator<Result> doAction(Action action) {
+ return Iterators.transform(blockingStub.doAction(action.toProtocol()), t -> new Result(t));
+ }
+
+ public void authenticateBasic(String username, String password) {
+ BasicClientAuthHandler basicClient = new BasicClientAuthHandler(username, password);
+ authenticate(basicClient);
+ }
+
+ public void authenticate(ClientAuthHandler handler) {
+ Preconditions.checkArgument(!authInterceptor.hasToken(), "Auth already completed.");
+ authInterceptor.setToken(ClientAuthWrapper.doClientAuth(handler, asyncStub));
+ }
+
+ /**
+ * Create or append a descriptor with another stream.
+ * @param descriptor
+ * @param root
+ * @return
+ */
+ public ClientStreamListener startPut(FlightDescriptor descriptor, VectorSchemaRoot root) {
+ Preconditions.checkNotNull(descriptor);
+ Preconditions.checkNotNull(root);
+
+ SetStreamObserver<PutResult> resultObserver = new SetStreamObserver<>();
+ ClientCallStreamObserver<ArrowMessage> observer = (ClientCallStreamObserver<ArrowMessage>)
+ asyncClientStreamingCall(channel.newCall(doPutDescriptor, asyncStub.getCallOptions()), resultObserver);
+ // send the schema to start.
+ ArrowMessage message = new ArrowMessage(descriptor.toProtocol(), root.getSchema());
+ observer.onNext(message);
+ return new PutObserver(new VectorUnloader(root, true, false), observer, resultObserver.getFuture());
+ }
+
+ public FlightInfo getInfo(FlightDescriptor descriptor) {
+ return new FlightInfo(blockingStub.getFlightInfo(descriptor.toProtocol()));
+ }
+
+ public FlightStream getStream(Ticket ticket) {
+ ClientCall<Flight.Ticket, ArrowMessage> call = channel.newCall(doGetDescriptor, asyncStub.getCallOptions());
+ FlightStream stream = new FlightStream(
+ allocator,
+ PENDING_REQUESTS,
+ (String message, Throwable cause) -> call.cancel(message, cause),
+ (count) -> call.request(count));
+
+ final StreamObserver<ArrowMessage> delegate = stream.asObserver();
+ ClientResponseObserver<Flight.Ticket, ArrowMessage> clientResponseObserver =
+ new ClientResponseObserver<Flight.Ticket, ArrowMessage>() {
+
+ @Override
+ public void beforeStart(ClientCallStreamObserver<org.apache.arrow.flight.impl.Flight.Ticket> requestStream) {
+ requestStream.disableAutoInboundFlowControl();
+ }
+
+ @Override
+ public void onNext(ArrowMessage value) {
+ delegate.onNext(value);
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ delegate.onError(t);
+ }
+
+ @Override
+ public void onCompleted() {
+ delegate.onCompleted();
+ }
+
+ };
+
+ asyncServerStreamingCall(call, ticket.toProtocol(), clientResponseObserver);
+ return stream;
+ }
+
+ private static class SetStreamObserver<T> implements StreamObserver<T> {
+ private final SettableFuture<T> result = SettableFuture.create();
+ private volatile T resultLocal;
+
+ @Override
+ public void onNext(T value) {
+ resultLocal = value;
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ result.setException(t);
+ }
+
+ @Override
+ public void onCompleted() {
+ result.set(Preconditions.checkNotNull(resultLocal));
+ }
+
+ public ListenableFuture<T> getFuture() {
+ return result;
+ }
+ }
+
+ private static class PutObserver implements ClientStreamListener {
+ private final ClientCallStreamObserver<ArrowMessage> observer;
+ private final VectorUnloader unloader;
+ private final ListenableFuture<PutResult> futureResult;
+
+ public PutObserver(VectorUnloader unloader, ClientCallStreamObserver<ArrowMessage> observer,
+ ListenableFuture<PutResult> futureResult) {
+ this.observer = observer;
+ this.unloader = unloader;
+ this.futureResult = futureResult;
+ }
+
+ @Override
+ public void putNext() {
+ ArrowRecordBatch batch = unloader.getRecordBatch();
+ while (!observer.isReady()) {
+ /* busy wait */
+ }
+ observer.onNext(new ArrowMessage(batch));
+ }
+
+ @Override
+ public void error(Throwable ex) {
+ observer.onError(ex);
+ }
+
+ @Override
+ public void completed() {
+ observer.onCompleted();
+ }
+
+ @Override
+ public PutResult getResult() {
+ try {
+ return futureResult.get();
+ } catch (Exception ex) {
+ throw Throwables.propagate(ex);
+ }
+ }
+ }
+
+ public interface ClientStreamListener {
+
+ public void putNext();
+
+ public void error(Throwable ex);
+
+ public void completed();
+
+ public PutResult getResult();
+
+ }
+
+
+ public void close() throws InterruptedException {
+ channel.shutdown().awaitTermination(5, TimeUnit.SECONDS);
+ allocator.close();
+ }
+
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightConstants.java
similarity index 64%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/FlightConstants.java
index a5b5c9d..6254665 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightConstants.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,10 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight;
-import com.google.common.base.Preconditions;
+public interface FlightConstants {
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+ public static final String SERVICE = "arrow.flight.protocol.FlightService";
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
- }
-
- public V getVector() {
- return vector;
- }
-
- public boolean isCreated() {
- return created;
- }
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightDescriptor.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightDescriptor.java
new file mode 100644
index 0000000..0f72312
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightDescriptor.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.List;
+
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.FlightDescriptor.DescriptorType;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
+
+public class FlightDescriptor {
+
+ private boolean isCmd;
+ private List<String> path;
+ private byte[] cmd;
+
+ private FlightDescriptor(boolean isCmd, List<String> path, byte[] cmd) {
+ super();
+ this.isCmd = isCmd;
+ this.path = path;
+ this.cmd = cmd;
+ }
+
+ public static FlightDescriptor command(byte[] cmd) {
+ return new FlightDescriptor(true, null, cmd);
+ }
+
+ public static FlightDescriptor path(Iterable<String> path) {
+ return new FlightDescriptor(false, ImmutableList.copyOf(path), null);
+ }
+
+ public static FlightDescriptor path(String...path) {
+ return new FlightDescriptor(false, ImmutableList.copyOf(path), null);
+ }
+
+ FlightDescriptor(Flight.FlightDescriptor descriptor) {
+ if (descriptor.getType() == DescriptorType.CMD) {
+ isCmd = true;
+ cmd = descriptor.getCmd().toByteArray();
+ } else if (descriptor.getType() == DescriptorType.PATH) {
+ isCmd = false;
+ path = descriptor.getPathList();
+ } else {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public boolean isCommand() {
+ return isCmd;
+ }
+
+ public List<String> getPath() {
+ Preconditions.checkArgument(!isCmd);
+ return path;
+ }
+
+ public byte[] getCommand() {
+ Preconditions.checkArgument(isCmd);
+ return cmd;
+ }
+
+ Flight.FlightDescriptor toProtocol() {
+ Flight.FlightDescriptor.Builder b = Flight.FlightDescriptor.newBuilder();
+
+ if (isCmd) {
+ return b.setType(DescriptorType.CMD).setCmd(ByteString.copyFrom(cmd)).build();
+ }
+ return b.setType(DescriptorType.PATH).addAllPath(path).build();
+ }
+
+ @Override
+ public String toString() {
+ if (isCmd) {
+ return toHex(cmd);
+ } else {
+ return Joiner.on('.').join(path);
+ }
+ }
+
+ private String toHex(byte[] bytes) {
+ StringBuilder sb = new StringBuilder();
+ for (byte b : bytes) {
+ sb.append(String.format("%02X ", b));
+ }
+ return sb.toString();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((cmd == null) ? 0 : cmd.hashCode());
+ result = prime * result + (isCmd ? 1231 : 1237);
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ FlightDescriptor other = (FlightDescriptor) obj;
+ if (cmd == null) {
+ if (other.cmd != null) {
+ return false;
+ }
+ } else if (!cmd.equals(other.cmd)) {
+ return false;
+ }
+ if (isCmd != other.isCmd) {
+ return false;
+ }
+ if (path == null) {
+ if (other.path != null) {
+ return false;
+ }
+ } else if (!path.equals(other.path)) {
+ return false;
+ }
+ return true;
+ }
+
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightEndpoint.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightEndpoint.java
new file mode 100644
index 0000000..28aeeb1
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightEndpoint.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.impl.Flight;
+
+import com.google.common.collect.ImmutableList;
+
+public class FlightEndpoint {
+ private List<Location> locations;
+ private Ticket ticket;
+
+ public FlightEndpoint(Ticket ticket, Location... locations) {
+ super();
+ this.locations = ImmutableList.copyOf(locations);
+ this.ticket = ticket;
+ }
+
+ public FlightEndpoint(Flight.FlightEndpoint flt) {
+ locations = flt.getLocationList().stream()
+ .map(t -> new Location(t)).collect(Collectors.toList());
+ ticket = new Ticket(flt.getTicket());
+ }
+
+ public List<Location> getLocations() {
+ return locations;
+ }
+
+ public Ticket getTicket() {
+ return ticket;
+ }
+
+ Flight.FlightEndpoint toProtocol() {
+ Flight.FlightEndpoint.Builder b = Flight.FlightEndpoint.newBuilder()
+ .setTicket(ticket.toProtocol());
+
+ for (Location l : locations) {
+ b.addLocation(Flight.Location.newBuilder()
+ .setHost(l.getHost())
+ .setPort(l.getPort())
+ .build());
+ }
+ return b.build();
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightInfo.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightInfo.java
new file mode 100644
index 0000000..d5a8e4f
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightInfo.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.FlightGetInfo;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.ByteString;
+
+public class FlightInfo {
+ private Schema schema;
+ private FlightDescriptor descriptor;
+ private List<FlightEndpoint> endpoints;
+ private final long bytes;
+ private final long records;
+
+ public FlightInfo(Schema schema, FlightDescriptor descriptor, List<FlightEndpoint> endpoints, long bytes,
+ long records) {
+ super();
+ this.schema = schema;
+ this.descriptor = descriptor;
+ this.endpoints = endpoints;
+ this.bytes = bytes;
+ this.records = records;
+ }
+
+ FlightInfo(FlightGetInfo flightGetInfo) {
+ schema = flightGetInfo.getSchema().size() > 0 ?
+ Schema.deserialize(flightGetInfo.getSchema().asReadOnlyByteBuffer()) : new Schema(ImmutableList.of());
+ descriptor = new FlightDescriptor(flightGetInfo.getFlightDescriptor());
+ endpoints = flightGetInfo.getEndpointList().stream().map(t -> new FlightEndpoint(t)).collect(Collectors.toList());
+ bytes = flightGetInfo.getTotalBytes();
+ records = flightGetInfo.getTotalRecords();
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public long getBytes() {
+ return bytes;
+ }
+
+ public long getRecords() {
+ return records;
+ }
+
+ public FlightDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ public List<FlightEndpoint> getEndpoints() {
+ return endpoints;
+ }
+
+ FlightGetInfo toProtocol() {
+ return Flight.FlightGetInfo.newBuilder()
+ .addAllEndpoint(endpoints.stream().map(t -> t.toProtocol()).collect(Collectors.toList()))
+ .setSchema(ByteString.copyFrom(schema.toByteArray()))
+ .setFlightDescriptor(descriptor.toProtocol())
+ .setTotalBytes(FlightInfo.this.bytes)
+ .setTotalRecords(records)
+ .build();
+
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightProducer.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightProducer.java
new file mode 100644
index 0000000..c21c0a9
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightProducer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.concurrent.Callable;
+
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+/**
+ * API to Implement an Arrow Flight server.
+ */
+public interface FlightProducer {
+
+ public void getStream(Ticket ticket, ServerStreamListener listener);
+
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener);
+
+ public FlightInfo getFlightInfo(FlightDescriptor descriptor);
+
+ public Callable<PutResult> acceptPut(FlightStream flightStream);
+
+ public Result doAction(Action action);
+
+ public void listActions(StreamListener<ActionType> listener);
+
+ public interface ServerStreamListener {
+
+ boolean isCancelled();
+
+ boolean isReady();
+
+ void start(VectorSchemaRoot root);
+
+ void putNext();
+
+ void error(Throwable ex);
+
+ void completed();
+
+ }
+
+ public interface StreamListener<T> {
+
+ void onNext(T val);
+
+ void onError(Throwable t);
+
+ void onCompleted();
+
+ }
+
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightServer.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightServer.java
new file mode 100644
index 0000000..66a6816
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightServer.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.auth.ServerAuthInterceptor;
+import org.apache.arrow.flight.impl.Flight.FlightGetInfo;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+
+import io.grpc.Server;
+import io.grpc.ServerBuilder;
+import io.grpc.ServerInterceptors;
+
+public class FlightServer implements AutoCloseable {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlightServer.class);
+
+ private final Server server;
+
+ public FlightServer(
+ BufferAllocator allocator,
+ int port,
+ FlightProducer producer,
+ ServerAuthHandler authHandler) {
+ this.server = ServerBuilder.forPort(port)
+ .addService(
+ ServerInterceptors.intercept(
+ new FlightBindingService(allocator, producer, authHandler),
+ new ServerAuthInterceptor(authHandler)))
+ .build();
+ }
+
+ public FlightServer start() throws IOException {
+ server.start();
+ return this;
+ }
+
+ public int getPort() {
+ return server.getPort();
+ }
+
+ public void close() throws InterruptedException {
+ server.shutdown();
+ final boolean terminated = server.awaitTermination(3000, TimeUnit.MILLISECONDS);
+ if (terminated) {
+ logger.debug("Server was terminated within 3s");
+ return;
+ }
+
+ // get more aggressive in termination.
+ server.shutdownNow();
+
+ int count = 0;
+ while (!server.isTerminated() & count < 30) {
+ count++;
+ logger.debug("Waiting for termination");
+ Thread.sleep(100);
+ }
+
+ if (!server.isTerminated()) {
+ logger.warn("Couldn't shutdown server, resources likely will be leaked.");
+ }
+ }
+
+ public interface OutputFlight {
+ void sendData(int count);
+
+ void done();
+
+ void fail(Throwable t);
+ }
+
+ public interface FlightServerHandler {
+
+ public FlightGetInfo getFlightInfo(String descriptor) throws Exception;
+
+ public OutputFlight setupFlight(VectorSchemaRoot root);
+
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightService.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightService.java
new file mode 100644
index 0000000..5f24684
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightService.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+import org.apache.arrow.flight.FlightProducer.ServerStreamListener;
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.auth.ServerAuthWrapper;
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.ActionType;
+import org.apache.arrow.flight.impl.Flight.Empty;
+import org.apache.arrow.flight.impl.Flight.FlightGetInfo;
+import org.apache.arrow.flight.impl.Flight.HandshakeRequest;
+import org.apache.arrow.flight.impl.Flight.HandshakeResponse;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.flight.impl.Flight.Result;
+import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceImplBase;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Preconditions;
+
+import io.grpc.stub.ServerCallStreamObserver;
+import io.grpc.stub.StreamObserver;
+
+class FlightService extends FlightServiceImplBase {
+
+ private static final Logger logger = LoggerFactory.getLogger(FlightService.class);
+ private final static int PENDING_REQUESTS = 5;
+
+ private final BufferAllocator allocator;
+ private final FlightProducer producer;
+ private final ServerAuthHandler authHandler;
+ private final ExecutorService executors = Executors.newCachedThreadPool();
+
+ public FlightService(BufferAllocator allocator, FlightProducer producer, ServerAuthHandler authHandler) {
+ this.allocator = allocator;
+ this.producer = producer;
+ this.authHandler = authHandler;
+ }
+
+ @Override
+ public StreamObserver<HandshakeRequest> handshake(StreamObserver<HandshakeResponse> responseObserver) {
+ return ServerAuthWrapper.wrapHandshake(authHandler, responseObserver, executors);
+ }
+
+ @Override
+ public void listFlights(Flight.Criteria criteria, StreamObserver<FlightGetInfo> responseObserver) {
+ try {
+ producer.listFlights(new Criteria(criteria), StreamPipe.wrap(responseObserver, t -> t.toProtocol()));
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ }
+
+ public void doGetCustom(Flight.Ticket ticket, StreamObserver<ArrowMessage> responseObserver) {
+ try {
+ producer.getStream(new Ticket(ticket), new GetListener(responseObserver));
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ }
+
+ @Override
+ public void doAction(Flight.Action request, StreamObserver<Result> responseObserver) {
+ try {
+ responseObserver.onNext(producer.doAction(new Action(request)).toProtocol());
+ responseObserver.onCompleted();
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ }
+
+ @Override
+ public void listActions(Empty request, StreamObserver<ActionType> responseObserver) {
+ try {
+ producer.listActions(StreamPipe.wrap(responseObserver, t -> t.toProtocol()));
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ }
+
+ private static class GetListener implements ServerStreamListener {
+ private ServerCallStreamObserver<ArrowMessage> responseObserver;
+ private volatile VectorUnloader unloader;
+
+ public GetListener(StreamObserver<ArrowMessage> responseObserver) {
+ super();
+ this.responseObserver = (ServerCallStreamObserver<ArrowMessage>) responseObserver;
+ this.responseObserver.setOnCancelHandler(() -> onCancel());
+ this.responseObserver.disableAutoInboundFlowControl();
+ }
+
+ private void onCancel() {
+ logger.debug("Stream cancelled by client.");
+ }
+
+ @Override
+ public boolean isReady() {
+
+ return responseObserver.isReady();
+ }
+
+ public boolean isCancelled() {
+ return responseObserver.isCancelled();
+ }
+
+ @Override
+ public void start(VectorSchemaRoot root) {
+ responseObserver.onNext(new ArrowMessage(null, root.getSchema()));
+ unloader = new VectorUnloader(root, true, false);
+ }
+
+ @Override
+ public void putNext() {
+ Preconditions.checkNotNull(unloader);
+ responseObserver.onNext(new ArrowMessage(unloader.getRecordBatch()));
+ }
+
+ @Override
+ public void error(Throwable ex) {
+ responseObserver.onError(ex);
+ }
+
+ @Override
+ public void completed() {
+ responseObserver.onCompleted();
+ }
+
+ }
+
+ public StreamObserver<ArrowMessage> doPutCustom(final StreamObserver<PutResult> responseObserverSimple) {
+ ServerCallStreamObserver<PutResult> responseObserver = (ServerCallStreamObserver<PutResult>) responseObserverSimple;
+ responseObserver.disableAutoInboundFlowControl();
+ responseObserver.request(1);
+
+ FlightStream fs = new FlightStream(allocator, PENDING_REQUESTS, null, (count) -> responseObserver.request(count));
+ executors.submit(() -> {
+ try {
+ responseObserver.onNext(producer.acceptPut(fs).call());
+ responseObserver.onCompleted();
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ });
+
+ return fs.asObserver();
+ }
+
+ @Override
+ public void getFlightInfo(Flight.FlightDescriptor request, StreamObserver<FlightGetInfo> responseObserver) {
+ try {
+ FlightInfo info = producer.getFlightInfo(new FlightDescriptor(request));
+ responseObserver.onNext(info.toProtocol());
+ responseObserver.onCompleted();
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/FlightStream.java b/java/flight/src/main/java/org/apache/arrow/flight/FlightStream.java
new file mode 100644
index 0000000..2029bea
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/FlightStream.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.SettableFuture;
+
+import io.grpc.stub.StreamObserver;
+
+public class FlightStream {
+
+
+ private final Object DONE = new Object();
+ private final Object DONE_EX = new Object();
+
+
+ private final BufferAllocator allocator;
+ private final Cancellable cancellable;
+ private final LinkedBlockingQueue<Object> queue = new LinkedBlockingQueue<>();
+ private final SettableFuture<VectorSchemaRoot> root = SettableFuture.create();
+ private final int pendingTarget;
+ private final Requestor requestor;
+
+ private volatile int pending = 1;
+ private boolean completed = false;
+ private volatile VectorSchemaRoot fulfilledRoot;
+ private volatile VectorLoader loader;
+ private volatile Throwable ex;
+ private volatile FlightDescriptor descriptor;
+ private volatile Schema schema;
+
+ public FlightStream(BufferAllocator allocator, int pendingTarget, Cancellable cancellable, Requestor requestor) {
+ this.allocator = allocator;
+ this.pendingTarget = pendingTarget;
+ this.cancellable = cancellable;
+ this.requestor = requestor;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ public FlightDescriptor getDescriptor() {
+ return descriptor;
+ }
+
+ public void close() throws Exception {
+ if (!completed && cancellable != null) {
+ cancel("Stream closed before end.", null);
+ }
+ List<AutoCloseable> closeables = ImmutableList.copyOf(queue.toArray()).stream()
+ .filter(t -> AutoCloseable.class.isAssignableFrom(t.getClass()))
+ .map(t -> ((AutoCloseable) t))
+ .collect(Collectors.toList());
+
+ AutoCloseables.close(Iterables.concat(closeables, ImmutableList.of(root.get())));
+ }
+
+ /**
+ * Blocking request to load next item into list.
+ * @return Whether or not more data was found.
+ */
+ public boolean next() {
+ try {
+ // make sure we have the root
+ root.get().clear();
+
+ if (completed && queue.isEmpty()) {
+ return false;
+ }
+
+
+ pending--;
+ requestOutstanding();
+
+ Object data = queue.take();
+ if (DONE == data) {
+ queue.put(DONE);
+ completed = true;
+ return false;
+ } else if (DONE_EX == data) {
+ queue.put(DONE_EX);
+ if (ex instanceof Exception) {
+ throw (Exception) ex;
+ } else {
+ throw new Exception(ex);
+ }
+ } else {
+ ArrowMessage msg = ((ArrowMessage) data);
+ try (ArrowRecordBatch arb = msg.asRecordBatch()) {
+ loader.load(arb);
+ }
+ return true;
+ }
+
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ public VectorSchemaRoot getRoot() {
+ try {
+ return root.get();
+ } catch (InterruptedException | ExecutionException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private synchronized void requestOutstanding() {
+ if (pending < pendingTarget) {
+ requestor.request(pendingTarget - pending);
+ pending = pendingTarget;
+ }
+ }
+
+ private class Observer implements StreamObserver<ArrowMessage> {
+
+ public Observer() {
+ super();
+ }
+
+ @Override
+ public void onNext(ArrowMessage msg) {
+ requestOutstanding();
+ switch (msg.getMessageType()) {
+ case SCHEMA:
+ schema = msg.asSchema();
+ fulfilledRoot = VectorSchemaRoot.create(schema, allocator);
+ loader = new VectorLoader(fulfilledRoot);
+ descriptor = msg.getDescriptor() != null ? new FlightDescriptor(msg.getDescriptor()) : null;
+ root.set(fulfilledRoot);
+
+ break;
+ case RECORD_BATCH:
+ queue.add(msg);
+ break;
+ case NONE:
+ case DICTIONARY_BATCH:
+ case TENSOR:
+ default:
+ queue.add(DONE_EX);
+ ex = new UnsupportedOperationException("Unable to handle message of type." + msg);
+
+ }
+
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ ex = t;
+ queue.add(DONE_EX);
+ }
+
+ @Override
+ public void onCompleted() {
+ queue.add(DONE);
+ }
+ }
+
+ public void cancel(String message, Throwable exception) {
+ if (cancellable != null) {
+ cancellable.cancel(message, exception);
+ } else {
+ throw new UnsupportedOperationException("Streams cannot be cancelled that are produced by client. " +
+ "Instead, server should reject incoming messages.");
+ }
+ }
+
+ StreamObserver<ArrowMessage> asObserver() {
+ return new Observer();
+ }
+
+ public interface Cancellable {
+ void cancel(String message, Throwable exception);
+ }
+
+ public interface Requestor {
+ void request(int count);
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/GenericOperation.java
similarity index 65%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/GenericOperation.java
index a5b5c9d..ca22d80 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/GenericOperation.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,25 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight;
-import com.google.common.base.Preconditions;
+class GenericOperation {
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+ private final String type;
+ private final byte[] body;
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
+ public GenericOperation(String type, byte[] body) {
+ super();
+ this.type = type;
+ this.body = body == null ? new byte[0] : body;
}
- public V getVector() {
- return vector;
+ public String getType() {
+ return type;
}
- public boolean isCreated() {
- return created;
+ public byte[] getBody() {
+ return body;
}
+
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/Location.java
similarity index 57%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/Location.java
index a5b5c9d..ea1bce9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/Location.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,36 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.flight.impl.Flight;
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+public class Location {
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
+ private final String host;
+ private final int port;
+
+ public Location(String host, int port) {
+ super();
+ this.host = host;
+ this.port = port;
+ }
+
+ Location(Flight.Location location) {
+ this.host = location.getHost();
+ this.port = location.getPort();
}
- public V getVector() {
- return vector;
+ public String getHost() {
+ return host;
}
- public boolean isCreated() {
- return created;
+ public int getPort() {
+ return port;
}
+
+ Flight.Location toProtocol() {
+ return Flight.Location.newBuilder().setHost(host).setPort(port).build();
+ }
+
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/NoOpFlightProducer.java b/java/flight/src/main/java/org/apache/arrow/flight/NoOpFlightProducer.java
new file mode 100644
index 0000000..efa05b2
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/NoOpFlightProducer.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.concurrent.Callable;
+
+import org.apache.arrow.flight.impl.Flight.PutResult;
+
+public class NoOpFlightProducer implements FlightProducer {
+
+ @Override
+ public void getStream(Ticket ticket, ServerStreamListener listener) {
+ listener.error(new UnsupportedOperationException("NYI"));
+ }
+
+ @Override
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener) {
+ listener.onError(new UnsupportedOperationException("NYI"));
+ }
+
+ @Override
+ public FlightInfo getFlightInfo(FlightDescriptor descriptor) {
+ throw new UnsupportedOperationException("NYI");
+ }
+
+ @Override
+ public Callable<PutResult> acceptPut(FlightStream flightStream) {
+ throw new UnsupportedOperationException("NYI");
+ }
+
+ @Override
+ public Result doAction(Action action) {
+ throw new UnsupportedOperationException("NYI");
+ }
+
+ @Override
+ public void listActions(StreamListener<ActionType> listener) {
+ listener.onError(new UnsupportedOperationException("NYI"));
+ }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/Result.java
similarity index 61%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/Result.java
index a5b5c9d..309231c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/Result.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,31 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.flight.impl.Flight;
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+import com.google.protobuf.ByteString;
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
+public class Result {
+
+ private final byte[] body;
+
+ public Result(byte[] body) {
+ this.body = body;
+ }
+
+ public Result(Flight.Result result) {
+ this.body = result.getBody().toByteArray();
}
- public V getVector() {
- return vector;
+ public byte[] getBody() {
+ return body;
}
- public boolean isCreated() {
- return created;
+ Flight.Result toProtocol() {
+ return Flight.Result.newBuilder()
+ .setBody(ByteString.copyFrom(body))
+ .build();
}
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/StreamPipe.java b/java/flight/src/main/java/org/apache/arrow/flight/StreamPipe.java
new file mode 100644
index 0000000..31d3635
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/StreamPipe.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.function.Function;
+
+import org.apache.arrow.flight.FlightProducer.StreamListener;
+
+import io.grpc.stub.StreamObserver;
+
+/**
+ * Shim listener to avoid exposing GRPC internals.
+
+ * @param <T>
+ */
+class StreamPipe<FROM, TO> implements StreamListener<FROM> {
+
+ private StreamObserver<TO> delegate;
+ private Function<FROM, TO> mapFunction;
+
+ public static <FROM, TO> StreamListener<FROM> wrap(StreamObserver<TO> delegate, Function<FROM, TO> func) {
+ return new StreamPipe<>(delegate, func);
+ }
+
+ public StreamPipe(StreamObserver<TO> delegate, Function<FROM, TO> func) {
+ super();
+ this.delegate = delegate;
+ this.mapFunction = func;
+ }
+
+ @Override
+ public void onNext(FROM val) {
+ delegate.onNext(mapFunction.apply(val));
+ }
+
+ @Override
+ public void onError(Throwable t) {
+ delegate.onError(t);
+ }
+
+ @Override
+ public void onCompleted() {
+ delegate.onCompleted();
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/Ticket.java b/java/flight/src/main/java/org/apache/arrow/flight/Ticket.java
new file mode 100644
index 0000000..858cc8d
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/Ticket.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.Arrays;
+
+import org.apache.arrow.flight.impl.Flight;
+
+import com.google.protobuf.ByteString;
+
+/**
+ * Endpoint for a particular stream.
+ */
+public class Ticket {
+ private final byte[] bytes;
+
+ public Ticket(byte[] bytes) {
+ super();
+ this.bytes = bytes;
+ }
+
+ public byte[] getBytes() {
+ return bytes;
+ }
+
+ Ticket(org.apache.arrow.flight.impl.Flight.Ticket ticket) {
+ this.bytes = ticket.getTicket().toByteArray();
+ }
+
+ Flight.Ticket toProtocol() {
+ return Flight.Ticket.newBuilder()
+ .setTicket(ByteString.copyFrom(bytes))
+ .build();
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + Arrays.hashCode(bytes);
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ Ticket other = (Ticket) obj;
+ if (!Arrays.equals(bytes, other.bytes)) {
+ return false;
+ }
+ return true;
+ }
+
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java
similarity index 52%
copy from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
copy to java/flight/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java
index 161b81a..a16bdde 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/AuthConstants.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,29 +16,31 @@
* limitations under the License.
*/
-package org.apache.arrow.memory;
+package org.apache.arrow.flight.auth;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.arrow.flight.FlightConstants;
-/**
- * The root allocator for using direct memory inside a Drillbit. Supports creating a
- * tree of descendant child allocators.
- */
-public class RootAllocator extends BaseAllocator {
-
- public RootAllocator(final long limit) {
- this(AllocationListener.NOOP, limit);
- }
-
- public RootAllocator(final AllocationListener listener, final long limit) {
- super(null, listener, "ROOT", 0, limit);
- }
-
- /**
- * Verify the accounting state of the allocation system.
- */
- @VisibleForTesting
- public void verify() {
- verifyAllocator();
- }
+import io.grpc.Metadata.BinaryMarshaller;
+import io.grpc.Metadata.Key;
+import io.grpc.MethodDescriptor;
+
+public final class AuthConstants {
+
+ public final static String HANDSHAKE_DESCRIPTOR_NAME = MethodDescriptor
+ .generateFullMethodName(FlightConstants.SERVICE, "Handshake");
+ public static final String TOKEN_NAME = "Auth-Token-bin";
+ public static final Key<byte[]> TOKEN_KEY = Key.of(TOKEN_NAME, new BinaryMarshaller<byte[]>() {
+
+ @Override
+ public byte[] toBytes(byte[] value) {
+ return value;
+ }
+
+ @Override
+ public byte[] parseBytes(byte[] serialized) {
+ return serialized;
+ }
+ });
+
+ private AuthConstants() {}
}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/BasicClientAuthHandler.java
similarity index 51%
copy from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
copy to java/flight/src/main/java/org/apache/arrow/flight/auth/BasicClientAuthHandler.java
index 161b81a..00c9b70 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/BasicClientAuthHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,29 +16,38 @@
* limitations under the License.
*/
-package org.apache.arrow.memory;
+package org.apache.arrow.flight.auth;
+
+import java.util.Iterator;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.arrow.flight.impl.Flight.BasicAuth;
/**
- * The root allocator for using direct memory inside a Drillbit. Supports creating a
- * tree of descendant child allocators.
+ * A client auth handler that supports username and password.
*/
-public class RootAllocator extends BaseAllocator {
+public class BasicClientAuthHandler implements ClientAuthHandler {
- public RootAllocator(final long limit) {
- this(AllocationListener.NOOP, limit);
- }
+ private final String name;
+ private final String password;
- public RootAllocator(final AllocationListener listener, final long limit) {
- super(null, listener, "ROOT", 0, limit);
+ public BasicClientAuthHandler(String name, String password) {
+ this.name = name;
+ this.password = password;
}
- /**
- * Verify the accounting state of the allocation system.
- */
- @VisibleForTesting
- public void verify() {
- verifyAllocator();
+ @Override
+ public byte[] authenticate(ClientAuthSender outgoing, Iterator<byte[]> incoming) {
+ BasicAuth.Builder builder = BasicAuth.newBuilder();
+ if (name != null) {
+ builder.setUsername(name);
+ }
+
+ if (password != null) {
+ builder.setPassword(password);
+ }
+
+ outgoing.send(builder.build().toByteArray());
+ return incoming.next();
}
+
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java
new file mode 100644
index 0000000..2bce89f
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/BasicServerAuthHandler.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import java.util.Iterator;
+
+import org.apache.arrow.flight.impl.Flight.BasicAuth;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * A ServerAuthHandler for username/password authentication.
+ */
+public class BasicServerAuthHandler implements ServerAuthHandler {
+
+ private static final Logger logger = LoggerFactory.getLogger(BasicServerAuthHandler.class);
+ private final BasicAuthValidator authValidator;
+
+ public BasicServerAuthHandler(BasicAuthValidator authValidator) {
+ super();
+ this.authValidator = authValidator;
+ }
+
+ public interface BasicAuthValidator {
+
+ public byte[] getToken(String username, String password) throws Exception;
+
+ public boolean isValid(byte[] token);
+
+ }
+
+ @Override
+ public boolean authenticate(ServerAuthSender outgoing, Iterator<byte[]> incoming) {
+ byte[] bytes = incoming.next();
+ try {
+ BasicAuth auth = BasicAuth.parseFrom(bytes);
+ byte[] token = authValidator.getToken(auth.getUsername(), auth.getPassword());
+ outgoing.send(token);
+ return true;
+ } catch (InvalidProtocolBufferException e) {
+ logger.debug("Failure parsing auth message.", e);
+ } catch (Exception e) {
+ logger.debug("Unknown error during authorization.", e);
+ }
+
+ return false;
+ }
+
+ @Override
+ public boolean isValid(byte[] token) {
+ return authValidator.isValid(token);
+ }
+
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthHandler.java
similarity index 64%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthHandler.java
index a5b5c9d..2214112 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,20 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.flight.auth;
-import com.google.common.base.Preconditions;
+import java.util.Iterator;
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
+public interface ClientAuthHandler {
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
- }
+ public byte[] authenticate(ClientAuthSender outgoing, Iterator<byte[]> incoming);
- public V getVector() {
- return vector;
- }
+ public interface ClientAuthSender {
+
+ public void send(byte[] payload);
+
+ public void onError(String message, Throwable cause);
- public boolean isCreated() {
- return created;
}
+
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthInterceptor.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthInterceptor.java
new file mode 100644
index 0000000..e4066f2
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthInterceptor.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import io.grpc.CallOptions;
+import io.grpc.Channel;
+import io.grpc.ClientCall;
+import io.grpc.ClientInterceptor;
+import io.grpc.ForwardingClientCall.SimpleForwardingClientCall;
+import io.grpc.Metadata;
+import io.grpc.MethodDescriptor;
+
+public class ClientAuthInterceptor implements ClientInterceptor {
+
+ private volatile Metadata headerWithAuth;
+
+ public void setToken(byte[] token) {
+ final Metadata md = new Metadata();
+ md.put(AuthConstants.TOKEN_KEY, token);
+ this.headerWithAuth = md;
+ }
+
+ public ClientAuthInterceptor() {
+ }
+
+ public boolean hasToken() {
+ return headerWithAuth != null;
+ }
+
+ @Override
+ public <ReqT, RespT> ClientCall<ReqT, RespT> interceptCall(MethodDescriptor<ReqT, RespT> methodDescriptor,
+ CallOptions callOptions, Channel next) {
+ ClientCall<ReqT, RespT> call = next.newCall(methodDescriptor, callOptions);
+
+ // once we have an auth header, add that to the calls.
+ if (headerWithAuth != null) {
+ call = new HeaderAttachingClientCall<>(call);
+ }
+
+ return call;
+ }
+
+ private final class HeaderAttachingClientCall<ReqT, RespT> extends SimpleForwardingClientCall<ReqT, RespT> {
+
+ private HeaderAttachingClientCall(ClientCall<ReqT, RespT> call) {
+ super(call);
+ }
+
+ @Override
+ public void start(Listener<RespT> responseListener, Metadata headers) {
+ headers.merge(headerWithAuth);
+ super.start(responseListener, headers);
+ }
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java
new file mode 100644
index 0000000..7b8f03b
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ClientAuthWrapper.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import java.util.Iterator;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.arrow.flight.auth.ClientAuthHandler.ClientAuthSender;
+import org.apache.arrow.flight.impl.Flight.HandshakeRequest;
+import org.apache.arrow.flight.impl.Flight.HandshakeResponse;
+import org.apache.arrow.flight.impl.FlightServiceGrpc.FlightServiceStub;
+
+import com.google.common.base.Throwables;
+import com.google.protobuf.ByteString;
+
+import io.grpc.stub.StreamObserver;
+
+public class ClientAuthWrapper {
+
+ /**
+ * Do client auth for a client.
+ * @param authHandler The handler to use.
+ * @param stub The service stub.
+ * @return The token if auth was successful.
+ */
+ public static byte[] doClientAuth(ClientAuthHandler authHandler, FlightServiceStub stub) {
+
+ AuthObserver observer = new AuthObserver();
+ observer.responseObserver = stub.handshake(observer);
+ byte[] bytes = authHandler.authenticate(observer.sender, observer.iter);
+ observer.responseObserver.onCompleted();
+ return bytes;
+ }
+
+ private static class AuthObserver implements StreamObserver<HandshakeResponse> {
+
+ private volatile StreamObserver<HandshakeRequest> responseObserver;
+ private final LinkedBlockingQueue<byte[]> messages = new LinkedBlockingQueue<>();
+ private final AuthSender sender = new AuthSender();
+ private volatile boolean completed = false;
+ private Throwable ex = null;
+
+ public AuthObserver() {
+ super();
+ }
+
+ @Override
+ public void onNext(HandshakeResponse value) {
+ ByteString payload = value.getPayload();
+ if (payload != null) {
+ messages.add(payload.toByteArray());
+ }
+ }
+
+ private Iterator<byte[]> iter = new Iterator<byte[]>() {
+
+ @Override
+ public byte[] next() {
+ while (ex == null && (!completed || !messages.isEmpty())) {
+ byte[] bytes = messages.poll();
+ if (bytes == null) {
+ // busy wait.
+ continue;
+ } else {
+ return bytes;
+ }
+ }
+
+ if (ex != null) {
+ throw Throwables.propagate(ex);
+ }
+
+ throw new IllegalStateException("You attempted to retrieve messages after there were none.");
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !messages.isEmpty();
+ }
+ };
+
+ @Override
+ public void onError(Throwable t) {
+ ex = t;
+ }
+
+ private class AuthSender implements ClientAuthSender {
+
+ @Override
+ public void send(byte[] payload) {
+ responseObserver.onNext(HandshakeRequest.newBuilder()
+ .setPayload(ByteString.copyFrom(payload))
+ .build());
+ }
+
+ @Override
+ public void onError(String message, Throwable cause) {
+ responseObserver.onError(cause);
+ }
+
+ }
+
+ @Override
+ public void onCompleted() {
+ completed = true;
+ }
+ }
+
+}
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthHandler.java
similarity index 56%
copy from java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
copy to java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthHandler.java
index 161b81a..52e30da 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthHandler.java
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -16,29 +16,34 @@
* limitations under the License.
*/
-package org.apache.arrow.memory;
+package org.apache.arrow.flight.auth;
-import com.google.common.annotations.VisibleForTesting;
+import java.util.Iterator;
-/**
- * The root allocator for using direct memory inside a Drillbit. Supports creating a
- * tree of descendant child allocators.
- */
-public class RootAllocator extends BaseAllocator {
+public interface ServerAuthHandler {
- public RootAllocator(final long limit) {
- this(AllocationListener.NOOP, limit);
- }
+ public boolean isValid(byte[] token);
- public RootAllocator(final AllocationListener listener, final long limit) {
- super(null, listener, "ROOT", 0, limit);
- }
+ public boolean authenticate(ServerAuthSender outgoing, Iterator<byte[]> incoming);
+
+ public interface ServerAuthSender {
+
+ public void send(byte[] payload);
+
+ public void onError(String message, Throwable cause);
- /**
- * Verify the accounting state of the allocation system.
- */
- @VisibleForTesting
- public void verify() {
- verifyAllocator();
}
+
+ ServerAuthHandler NO_OP = new ServerAuthHandler() {
+
+ @Override
+ public boolean isValid(byte[] token) {
+ return true;
+ }
+
+ @Override
+ public boolean authenticate(ServerAuthSender outgoing, Iterator<byte[]> incoming) {
+ return true;
+ }
+ };
}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthInterceptor.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthInterceptor.java
new file mode 100644
index 0000000..1022504
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthInterceptor.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import io.grpc.Metadata;
+import io.grpc.ServerCall;
+import io.grpc.ServerCall.Listener;
+import io.grpc.ServerCallHandler;
+import io.grpc.ServerInterceptor;
+import io.grpc.Status;
+
+public class ServerAuthInterceptor implements ServerInterceptor {
+
+ private final ServerAuthHandler authHandler;
+
+ public ServerAuthInterceptor(ServerAuthHandler authHandler) {
+ this.authHandler = authHandler;
+ }
+
+ @Override
+ public <ReqT, RespT> Listener<ReqT> interceptCall(ServerCall<ReqT, RespT> call, Metadata headers,
+ ServerCallHandler<ReqT, RespT> next) {
+ if (
+ !call.getMethodDescriptor().getFullMethodName().equals(AuthConstants.HANDSHAKE_DESCRIPTOR_NAME) &&
+ !isValid(headers)) {
+ call.close(Status.PERMISSION_DENIED, new Metadata());
+ // TODO: we should actually terminate here instead of causing an exception below.
+ return new NoopServerCallListener<>();
+ }
+
+ return next.startCall(call, headers);
+ }
+
+ private final boolean isValid(Metadata headers) {
+ byte[] token = headers.get(AuthConstants.TOKEN_KEY);
+ return authHandler.isValid(token);
+ }
+
+ private static class NoopServerCallListener<T> extends ServerCall.Listener<T> {
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java
new file mode 100644
index 0000000..eba5c94
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/auth/ServerAuthWrapper.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import java.util.Iterator;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+
+import org.apache.arrow.flight.auth.ServerAuthHandler.ServerAuthSender;
+import org.apache.arrow.flight.impl.Flight.HandshakeRequest;
+import org.apache.arrow.flight.impl.Flight.HandshakeResponse;
+
+import com.google.protobuf.ByteString;
+
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+
+public class ServerAuthWrapper {
+
+ /**
+ * Wrap the auth handler for handshake purposes.
+ *
+ * @param authHandler
+ * @param responseObserver
+ * @param executors
+ * @return
+ */
+ public static StreamObserver<HandshakeRequest> wrapHandshake(ServerAuthHandler authHandler,
+ StreamObserver<HandshakeResponse> responseObserver, ExecutorService executors) {
+
+ // stream started.
+ AuthObserver observer = new AuthObserver(responseObserver);
+ final Runnable r = () -> {
+ try {
+ if (authHandler.authenticate(observer.sender, observer.iter)) {
+ responseObserver.onCompleted();
+ return;
+ }
+
+ responseObserver.onError(Status.PERMISSION_DENIED.asException());
+ } catch (Exception ex) {
+ responseObserver.onError(ex);
+ }
+ };
+ observer.future = executors.submit(r);
+ return observer;
+ }
+
+ private static class AuthObserver implements StreamObserver<HandshakeRequest> {
+
+ private final StreamObserver<HandshakeResponse> responseObserver;
+ private volatile Future<?> future;
+ private volatile boolean completed = false;
+ private final LinkedBlockingQueue<byte[]> messages = new LinkedBlockingQueue<>();
+ private final AuthSender sender = new AuthSender();
+
+ public AuthObserver(StreamObserver<HandshakeResponse> responseObserver) {
+ super();
+ this.responseObserver = responseObserver;
+ }
+
+ @Override
+ public void onNext(HandshakeRequest value) {
+ ByteString payload = value.getPayload();
+ if (payload != null) {
+ messages.add(payload.toByteArray());
+ }
+ }
+
+ private Iterator<byte[]> iter = new Iterator<byte[]>() {
+
+ @Override
+ public byte[] next() {
+ while (!completed || !messages.isEmpty()) {
+ byte[] bytes = messages.poll();
+ if (bytes == null) {
+ //busy wait.
+ continue;
+ }
+ return bytes;
+ }
+ throw new IllegalStateException("Requesting more messages than client sent.");
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !messages.isEmpty();
+ }
+ };
+
+ @Override
+ public void onError(Throwable t) {
+ while (future == null) {/* busy wait */}
+ future.cancel(true);
+ }
+
+ @Override
+ public void onCompleted() {
+ completed = true;
+ }
+
+ private class AuthSender implements ServerAuthSender {
+
+ @Override
+ public void send(byte[] payload) {
+ responseObserver.onNext(HandshakeResponse.newBuilder()
+ .setPayload(ByteString.copyFrom(payload))
+ .build());
+ }
+
+ @Override
+ public void onError(String message, Throwable cause) {
+ responseObserver.onError(cause);
+ }
+
+ }
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleFlightServer.java b/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleFlightServer.java
new file mode 100644
index 0000000..191b3ce
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleFlightServer.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.io.IOException;
+
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+
+/**
+ * An Example Flight Server that provides access to the InMemoryStore
+ */
+public class ExampleFlightServer implements AutoCloseable {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExampleFlightServer.class);
+
+ private final FlightServer flightServer;
+ private final Location location;
+ private final BufferAllocator allocator;
+ private final InMemoryStore mem;
+
+ public ExampleFlightServer(BufferAllocator allocator, Location location) {
+ this.allocator = allocator.newChildAllocator("flight-server", 0, Long.MAX_VALUE);
+ this.location = location;
+ this.mem = new InMemoryStore(this.allocator, location);
+ this.flightServer = new FlightServer(allocator, location.getPort(), mem, ServerAuthHandler.NO_OP);
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ public void start() throws IOException {
+ flightServer.start();
+ }
+
+ public InMemoryStore getStore() {
+ return mem;
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(mem, flightServer, allocator);
+ }
+
+ public static void main(String[] args) throws Exception {
+ final BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
+ final ExampleFlightServer efs = new ExampleFlightServer(a, new Location("localhost", 12233));
+ efs.start();
+ Runtime.getRuntime().addShutdownHook(new Thread(() -> {
+ try {
+ System.out.println("\nExiting...");
+ AutoCloseables.close(efs, a);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }));
+ while (true) {
+ Thread.sleep(30000);
+ }
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleTicket.java b/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleTicket.java
new file mode 100644
index 0000000..bcaa9a1
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/example/ExampleTicket.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.arrow.flight.Ticket;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+
+@JsonSerialize
+public class ExampleTicket {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private final List<String> path;
+ private final int ordinal;
+
+ // uuid to ensure that a stream from one node is not recreated on another node and mixed up.
+ private final String uuid;
+
+ @JsonCreator
+ public ExampleTicket(@JsonProperty("path") List<String> path, @JsonProperty("ordinal") int ordinal,
+ @JsonProperty("uuid") String uuid) {
+ super();
+ Preconditions.checkArgument(ordinal >= 0);
+ this.path = path;
+ this.ordinal = ordinal;
+ this.uuid = uuid;
+ }
+
+ public List<String> getPath() {
+ return path;
+ }
+
+ public int getOrdinal() {
+ return ordinal;
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public static ExampleTicket from(Ticket ticket) {
+ try {
+ return MAPPER.readValue(ticket.getBytes(), ExampleTicket.class);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ public Ticket toTicket() {
+ try {
+ return new Ticket(MAPPER.writeValueAsBytes(this));
+ } catch (JsonProcessingException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ordinal;
+ result = prime * result + ((path == null) ? 0 : path.hashCode());
+ result = prime * result + ((uuid == null) ? 0 : uuid.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ ExampleTicket other = (ExampleTicket) obj;
+ if (ordinal != other.ordinal) {
+ return false;
+ }
+ if (path == null) {
+ if (other.path != null) {
+ return false;
+ }
+ } else if (!path.equals(other.path)) {
+ return false;
+ }
+ if (uuid == null) {
+ if (other.uuid != null) {
+ return false;
+ }
+ } else if (!uuid.equals(other.uuid)) {
+ return false;
+ }
+ return true;
+ }
+
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/example/FlightHolder.java b/java/flight/src/main/java/org/apache/arrow/flight/example/FlightHolder.java
new file mode 100644
index 0000000..12cff26
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/example/FlightHolder.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+
+public class FlightHolder implements AutoCloseable {
+
+ private final BufferAllocator allocator;
+ private final FlightDescriptor descriptor;
+ private final Schema schema;
+ private final List<Stream> streams = new CopyOnWriteArrayList<>();
+
+ public FlightHolder(BufferAllocator allocator, FlightDescriptor descriptor, Schema schema) {
+ Preconditions.checkArgument(!descriptor.isCommand());
+ this.allocator = allocator.newChildAllocator(descriptor.toString(), 0, Long.MAX_VALUE);
+ this.descriptor = descriptor;
+ this.schema = schema;
+ }
+
+ public Stream getStream(ExampleTicket ticket) {
+ Preconditions.checkArgument(ticket.getOrdinal() < streams.size(), "Unknown stream.");
+ Stream stream = streams.get(ticket.getOrdinal());
+ stream.verify(ticket);
+ return stream;
+ }
+
+ public Stream.StreamCreator addStream(Schema schema) {
+ Preconditions.checkArgument(schema.equals(schema), "Stream schema inconsitent with existing schema.");
+ return new Stream.StreamCreator(schema, allocator, t -> {
+ synchronized (streams) {
+ streams.add(t);
+ }
+ });
+ }
+
+ public FlightInfo getFlightInfo(final Location l) {
+ final long bytes = allocator.getAllocatedMemory();
+ final long records = streams.stream().collect(Collectors.summingLong(t -> t.getRecordCount()));
+
+ final List<FlightEndpoint> endpoints = new ArrayList<>();
+ int i = 0;
+ for (Stream s : streams) {
+ endpoints.add(
+ new FlightEndpoint(
+ new ExampleTicket(descriptor.getPath(), i, s.getUuid())
+ .toTicket(),
+ l));
+ i++;
+ }
+ return new FlightInfo(schema, descriptor, endpoints, bytes, records);
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(Iterables.concat(streams, ImmutableList.of(allocator)));
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/example/InMemoryStore.java b/java/flight/src/main/java/org/apache/arrow/flight/example/InMemoryStore.java
new file mode 100644
index 0000000..dd0d47d
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/example/InMemoryStore.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.arrow.flight.Action;
+import org.apache.arrow.flight.ActionType;
+import org.apache.arrow.flight.Criteria;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightProducer;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.Result;
+import org.apache.arrow.flight.Ticket;
+import org.apache.arrow.flight.example.Stream.StreamCreator;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.VectorUnloader;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+/**
+ * A FlightProducer that hosts an in memory store of Arrow buffers.
+ */
+public class InMemoryStore implements FlightProducer, AutoCloseable {
+
+ private final ConcurrentMap<FlightDescriptor, FlightHolder> holders = new ConcurrentHashMap<>();
+ private final BufferAllocator allocator;
+ private final Location location;
+
+ public InMemoryStore(BufferAllocator allocator, Location location) {
+ super();
+ this.allocator = allocator;
+ this.location = location;
+ }
+
+ @Override
+ public void getStream(Ticket ticket, ServerStreamListener listener) {
+ getStream(ticket).sendTo(allocator, listener);
+ }
+
+ public Stream getStream(Ticket t) {
+ ExampleTicket example = ExampleTicket.from(t);
+ FlightDescriptor d = FlightDescriptor.path(example.getPath());
+ FlightHolder h = holders.get(d);
+ if (h == null) {
+ throw new IllegalStateException("Unknown ticket.");
+ }
+
+ return h.getStream(example);
+ }
+
+ public StreamCreator putStream(final FlightDescriptor descriptor, final Schema schema) {
+ final FlightHolder h = holders.computeIfAbsent(
+ descriptor,
+ t -> new FlightHolder(allocator, t, schema));
+
+ return h.addStream(schema);
+ }
+
+ @Override
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener) {
+ try {
+ for (FlightHolder h : holders.values()) {
+ listener.onNext(h.getFlightInfo(location));
+ }
+ listener.onCompleted();
+ } catch (Exception ex) {
+ listener.onError(ex);
+ }
+ }
+
+ @Override
+ public FlightInfo getFlightInfo(FlightDescriptor descriptor) {
+ FlightHolder h = holders.get(descriptor);
+ if (h == null) {
+ throw new IllegalStateException("Unknown descriptor.");
+ }
+
+ return h.getFlightInfo(location);
+ }
+
+ @Override
+ public Callable<PutResult> acceptPut(final FlightStream flightStream) {
+ return () -> {
+ StreamCreator creator = null;
+ boolean success = false;
+ try (VectorSchemaRoot root = flightStream.getRoot()) {
+ final FlightHolder h = holders.computeIfAbsent(
+ flightStream.getDescriptor(),
+ t -> new FlightHolder(allocator, t, flightStream.getSchema()));
+
+ creator = h.addStream(flightStream.getSchema());
+
+ VectorUnloader unloader = new VectorUnloader(root);
+ while (flightStream.next()) {
+ creator.add(unloader.getRecordBatch());
+ }
+ creator.complete();
+ success = true;
+ return PutResult.getDefaultInstance();
+ } finally {
+ if (!success) {
+ creator.drop();
+ }
+ }
+
+ };
+
+ }
+
+ @Override
+ public Result doAction(Action action) {
+ switch (action.getType()) {
+ case "drop":
+ return new Result(new byte[0]);
+ // not implemented.
+ }
+
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void listActions(StreamListener<ActionType> listener) {
+ listener.onNext(new ActionType("get", "pull a stream. Action must be done via standard get mechanism"));
+ listener.onNext(new ActionType("put", "push a stream. Action must be done via standard get mechanism"));
+ listener.onNext(new ActionType("drop", "delete a flight. Action body is a JSON encoded path."));
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(holders.values());
+ holders.clear();
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/example/Stream.java b/java/flight/src/main/java/org/apache/arrow/flight/example/Stream.java
new file mode 100644
index 0000000..48df2a3
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/example/Stream.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.UUID;
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.FlightProducer.ServerStreamListener;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.VectorLoader;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+
+public class Stream implements AutoCloseable, Iterable<ArrowRecordBatch> {
+
+ private final String uuid = UUID.randomUUID().toString();
+ private final List<ArrowRecordBatch> batches;
+ private final Schema schema;
+ private final long recordCount;
+
+ public Stream(
+ final Schema schema,
+ List<ArrowRecordBatch> batches,
+ long recordCount) {
+ this.schema = schema;
+ this.batches = ImmutableList.copyOf(batches);
+ this.recordCount = recordCount;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+
+ @Override
+ public Iterator<ArrowRecordBatch> iterator() {
+ return batches.iterator();
+ }
+
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ public String getUuid() {
+ return uuid;
+ }
+
+ public void sendTo(BufferAllocator allocator, ServerStreamListener listener) {
+ try (VectorSchemaRoot root = VectorSchemaRoot.create(schema, allocator)) {
+ listener.start(root);
+ final VectorLoader loader = new VectorLoader(root);
+ for (ArrowRecordBatch batch : batches) {
+ loader.load(batch);
+ listener.putNext();
+ }
+ listener.completed();
+ } catch (Exception ex) {
+ listener.error(ex);
+ }
+ }
+
+ public void verify(ExampleTicket ticket) {
+ if (!uuid.equals(ticket.getUuid())) {
+ throw new IllegalStateException("Ticket doesn't match.");
+ }
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(batches);
+ }
+
+ public static class StreamCreator {
+
+ private final Schema schema;
+ private final BufferAllocator allocator;
+ private final List<ArrowRecordBatch> batches = new ArrayList<>();
+ private final Consumer<Stream> committer;
+ private long recordCount = 0;
+
+ public StreamCreator(Schema schema, BufferAllocator allocator, Consumer<Stream> committer) {
+ this.allocator = allocator;
+ this.committer = committer;
+ this.schema = schema;
+ }
+
+ public void drop() {
+ try {
+ AutoCloseables.close(batches);
+ } catch (Exception ex) {
+ throw Throwables.propagate(ex);
+ }
+ }
+
+ public void add(ArrowRecordBatch batch) {
+ batches.add(batch.cloneWithTransfer(allocator));
+ recordCount += batch.getLength();
+ }
+
+ public void complete() {
+ Stream stream = new Stream(schema, batches, recordCount);
+ committer.accept(stream);
+ }
+
+ }
+
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java b/java/flight/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java
new file mode 100644
index 0000000..4844daa
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/grpc/AddWritableBuffer.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.grpc;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.List;
+
+import io.netty.buffer.ByteBuf;
+
+/**
+ * Allow a user to add a ByteBuf based InputStream directly into GRPC WritableBuffer to avoid an
+ * extra copy. This could be solved in GRPC by adding a ByteBufListable interface on InputStream and
+ * letting BufferChainOutputStream take adavantage of it.
+ */
+public class AddWritableBuffer {
+
+ private static final Constructor<?> bufConstruct;
+ private static final Field bufferList;
+ private static final Field current;
+ private static final Method listAdd;
+ private static final Class<?> bufChainOut;
+
+ static {
+
+ Constructor<?> tmpConstruct = null;
+ Field tmpBufferList = null;
+ Field tmpCurrent = null;
+ Class<?> tmpBufChainOut = null;
+ Method tmpListAdd = null;
+
+ try {
+ Class<?> nwb = Class.forName("io.grpc.netty.NettyWritableBuffer");
+
+ Constructor<?> tmpConstruct2 = nwb.getDeclaredConstructor(ByteBuf.class);
+ tmpConstruct2.setAccessible(true);
+
+ Class<?> tmpBufChainOut2 = Class.forName("io.grpc.internal.MessageFramer$BufferChainOutputStream");
+
+ Field tmpBufferList2 = tmpBufChainOut2.getDeclaredField("bufferList");
+ tmpBufferList2.setAccessible(true);
+
+ Field tmpCurrent2 = tmpBufChainOut2.getDeclaredField("current");
+ tmpCurrent2.setAccessible(true);
+
+ Method tmpListAdd2 = List.class.getDeclaredMethod("add", Object.class);
+
+ // output fields last.
+ tmpConstruct = tmpConstruct2;
+ tmpBufferList = tmpBufferList2;
+ tmpCurrent = tmpCurrent2;
+ tmpListAdd = tmpListAdd2;
+ tmpBufChainOut = tmpBufChainOut2;
+
+ } catch (Exception ex) {
+ ex.printStackTrace();
+ }
+
+ bufConstruct = tmpConstruct;
+ bufferList = tmpBufferList;
+ current = tmpCurrent;
+ listAdd = tmpListAdd;
+ bufChainOut = tmpBufChainOut;
+
+ }
+
+ /**
+ * Add the provided ByteBuf to the output stream if it is possible.
+ * @param buf The buffer to add.
+ * @param stream The Candidate OutputStream to add to.
+ * @return True if added. False if not possible.
+ * @throws IOException
+ */
+ public static boolean add(ByteBuf buf, OutputStream stream) throws IOException {
+ buf.readBytes(stream, buf.readableBytes());
+
+ if (bufChainOut == null) {
+ return false;
+ }
+
+ if (!stream.getClass().equals(bufChainOut)) {
+ return false;
+ }
+
+ try {
+ if (current.get(stream) != null) {
+ return false;
+ }
+
+ buf.retain();
+ Object obj = bufConstruct.newInstance(buf);
+ Object list = bufferList.get(stream);
+ listAdd.invoke(list, obj);
+ current.set(stream, obj);
+ return true;
+ } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | InstantiationException e) {
+ e.printStackTrace();
+ return false;
+ }
+ }
+}
diff --git a/java/flight/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java b/java/flight/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java
new file mode 100644
index 0000000..79db9fe
--- /dev/null
+++ b/java/flight/src/main/java/org/apache/arrow/flight/grpc/GetReadableBuffer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.grpc;
+
+import java.io.InputStream;
+import java.lang.reflect.Field;
+
+import com.google.common.base.Throwables;
+
+import io.grpc.internal.ReadableBuffer;
+
+/**
+ * Enable access to ReadableBuffer directly to copy data from an BufferInputStream into a target
+ * ByteBuffer/ByteBuf.
+ *
+ * This could be solved by BufferInputStream exposing Drainable.
+ */
+public class GetReadableBuffer {
+
+ private static final Field READABLE_BUFFER;
+ private static final Class<?> BUFFER_INPUT_STREAM;
+
+ static {
+ Field tmpField = null;
+ Class<?> tmpClazz = null;
+ try {
+ Class<?> clazz = Class.forName("io.grpc.internal.ReadableBuffers$BufferInputStream");
+
+ Field f = clazz.getDeclaredField("buffer");
+ f.setAccessible(true);
+ // don't set until we've gotten past all exception cases.
+ tmpField = f;
+ tmpClazz = clazz;
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ READABLE_BUFFER = tmpField;
+ BUFFER_INPUT_STREAM = tmpClazz;
+ }
+
+ public static ReadableBuffer getReadableBuffer(InputStream is) {
+
+ if (BUFFER_INPUT_STREAM == null || !is.getClass().equals(BUFFER_INPUT_STREAM)) {
+ return null;
+ }
+
+ try {
+ return (ReadableBuffer) READABLE_BUFFER.get(is);
+ } catch (Exception ex) {
+ throw Throwables.propagate(ex);
+ }
+ }
+
+}
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/TestBackPressure.java b/java/flight/src/test/java/org/apache/arrow/flight/TestBackPressure.java
new file mode 100644
index 0000000..9a6d83d
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/TestBackPressure.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.perf.PerformanceTestServer;
+import org.apache.arrow.flight.perf.TestPerf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestBackPressure {
+
+ private static final int BATCH_SIZE = 4095;
+
+ /**
+ * Make sure that failing to consume one stream doesn't block other streams.
+ */
+ @Test
+ public void ensureIndependentSteams() throws Exception {
+
+ final Location l = new Location("localhost", 12233);
+ try (
+ final BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
+ final PerformanceTestServer server = new PerformanceTestServer(a, l);
+ final FlightClient client = new FlightClient(a, l);
+ ) {
+
+ server.start();
+
+ FlightStream fs1 = client.getStream(client.getInfo(
+ TestPerf.getPerfFlightDescriptor(110l * BATCH_SIZE, BATCH_SIZE, 1))
+ .getEndpoints().get(0).getTicket());
+ consume(fs1, 10);
+
+ // stop consuming fs1 but make sure we can consume a large amount of fs2.
+ FlightStream fs2 = client.getStream(client.getInfo(
+ TestPerf.getPerfFlightDescriptor(200l * BATCH_SIZE, BATCH_SIZE, 1))
+ .getEndpoints().get(0).getTicket());
+ consume(fs2, 100);
+
+ consume(fs1, 100);
+ consume(fs2, 100);
+
+ consume(fs1);
+ consume(fs2);
+
+ fs1.close();
+ fs2.close();
+
+ }
+ }
+
+ /**
+ * Make sure that a stream doesn't go faster than the consumer is consuming.
+ */
+ @Test
+ public void ensureWaitUntilProceed() throws Exception {
+ // request some values.
+ final long wait = 3000;
+ final long epsilon = 1000;
+
+ final Location l = new Location("localhost", 12233);
+ AtomicLong sleepTime = new AtomicLong(0);
+ try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE)) {
+
+ final FlightProducer producer = new NoOpFlightProducer() {
+
+ @Override
+ public void getStream(Ticket ticket, ServerStreamListener listener) {
+ int batches = 0;
+ final Schema pojoSchema = new Schema(ImmutableList.of(Field.nullable("a", MinorType.BIGINT.getType())));
+ VectorSchemaRoot root = VectorSchemaRoot.create(pojoSchema, allocator);
+ listener.start(root);
+ while (true) {
+ while (!listener.isReady()) {
+ try {
+ Thread.sleep(1);
+ sleepTime.addAndGet(1l);
+ } catch (InterruptedException e) {
+ }
+ }
+
+ if (batches > 100) {
+ root.clear();
+ listener.completed();
+ return;
+ }
+
+ root.allocateNew();
+ root.setRowCount(4095);
+ listener.putNext();
+ batches++;
+ }
+ }
+ };
+
+
+ try (
+ BufferAllocator serverAllocator = allocator.newChildAllocator("server", 0, Long.MAX_VALUE);
+ FlightServer server = new FlightServer(serverAllocator, l.getPort(), producer, ServerAuthHandler.NO_OP);
+ BufferAllocator clientAllocator = allocator.newChildAllocator("client", 0, Long.MAX_VALUE);
+ FlightClient client = new FlightClient(clientAllocator, l)
+ ) {
+
+ server.start();
+ FlightStream stream = client.getStream(new Ticket(new byte[1]));
+ VectorSchemaRoot root = stream.getRoot();
+ root.clear();
+ Thread.sleep(wait);
+ while (stream.next()) {
+ root.clear();
+ }
+ long expected = wait - epsilon;
+ Assert.assertTrue(
+ String.format("Expected a sleep of at least %dms but only slept for %d", expected, sleepTime.get()),
+ sleepTime.get() > expected);
+
+ }
+ }
+ }
+
+ private static void consume(FlightStream stream) {
+ VectorSchemaRoot root = stream.getRoot();
+ while (stream.next()) {
+ root.clear();
+ }
+ }
+
+ private static void consume(FlightStream stream, int batches) {
+ VectorSchemaRoot root = stream.getRoot();
+ while (batches > 0 && stream.next()) {
+ root.clear();
+ batches--;
+ }
+ }
+}
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/TestBasicOperation.java b/java/flight/src/test/java/org/apache/arrow/flight/TestBasicOperation.java
new file mode 100644
index 0000000..bd82d09
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/TestBasicOperation.java
@@ -0,0 +1,257 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight;
+
+import java.util.concurrent.Callable;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+
+import org.apache.arrow.flight.FlightClient.ClientStreamListener;
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.impl.Flight;
+import org.apache.arrow.flight.impl.Flight.FlightDescriptor.DescriptorType;
+import org.apache.arrow.flight.impl.Flight.FlightGetInfo;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.google.common.base.Charsets;
+import com.google.protobuf.ByteString;
+
+/**
+ * Test the operations of a basic flight service.
+ */
+public class TestBasicOperation {
+
+ @Test
+ public void getDescriptors() throws Exception {
+ test(c -> {
+ for (FlightInfo i : c.listFlights(Criteria.ALL)) {
+ System.out.println(i.getDescriptor());
+ }
+ });
+ }
+
+ @Test
+ public void getDescriptor() throws Exception {
+ test(c -> {
+ System.out.println(c.getInfo(FlightDescriptor.path("hello")).getDescriptor());
+ });
+ }
+
+ @Test
+ public void listActions() throws Exception {
+ test(c -> {
+ for (ActionType at : c.listActions()) {
+ System.out.println(at.getType());
+ }
+ });
+ }
+
+ @Test
+ public void doAction() throws Exception {
+ test(c -> {
+ Result r = c.doAction(new Action("hello")).next();
+ System.out.println(new String(r.getBody(), Charsets.UTF_8));
+ });
+ }
+
+ @Test
+ public void putStream() throws Exception {
+ test((c, a) -> {
+ final int size = 10;
+
+ IntVector iv = new IntVector("c1", a);
+
+ VectorSchemaRoot root = VectorSchemaRoot.of(iv);
+ ClientStreamListener listener = c.startPut(FlightDescriptor.path("hello"), root);
+
+ //batch 1
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+
+ // batch 2
+
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i + size);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+ root.clear();
+ listener.completed();
+
+ // wait for ack to avoid memory leaks.
+ listener.getResult();
+ });
+ }
+
+
+ @Test
+ public void getStream() throws Exception {
+ test(c -> {
+ FlightStream stream = c.getStream(new Ticket(new byte[0]));
+ VectorSchemaRoot root = stream.getRoot();
+ IntVector iv = (IntVector) root.getVector("c1");
+ int value = 0;
+ while (stream.next()) {
+ for (int i = 0; i < root.getRowCount(); i++) {
+ Assert.assertEquals(value, iv.get(i));
+ value++;
+ }
+ }
+ });
+ }
+
+ private void test(Consumer<FlightClient> consumer) throws Exception {
+ test((c, a) -> {
+ consumer.accept(c);
+ });
+ }
+
+ private void test(BiConsumer<FlightClient, BufferAllocator> consumer) throws Exception {
+ try (
+ BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
+ Producer producer = new Producer(a);
+ FlightServer s = new FlightServer(a, 12233, producer, ServerAuthHandler.NO_OP);) {
+
+ s.start();
+
+ try (
+ FlightClient c = new FlightClient(a, new Location("localhost", 12233));
+ ) {
+ try (BufferAllocator testAllocator = a.newChildAllocator("testcase", 0, Long.MAX_VALUE)) {
+ consumer.accept(c, testAllocator);
+ }
+ }
+ }
+ }
+
+ /**
+ * An example FlightProducer for test purposes.
+ */
+ public class Producer implements FlightProducer, AutoCloseable {
+
+ private final BufferAllocator allocator;
+
+ public Producer(BufferAllocator allocator) {
+ super();
+ this.allocator = allocator;
+ }
+
+ @Override
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener) {
+ FlightGetInfo getInfo = FlightGetInfo.newBuilder()
+ .setFlightDescriptor(Flight.FlightDescriptor.newBuilder()
+ .setType(DescriptorType.CMD)
+ .setCmd(ByteString.copyFrom("cool thing", Charsets.UTF_8)))
+ .build();
+ listener.onNext(new FlightInfo(getInfo));
+ listener.onCompleted();
+ }
+
+ @Override
+ public Callable<PutResult> acceptPut(FlightStream flightStream) {
+ return () -> {
+ try (VectorSchemaRoot root = flightStream.getRoot()) {
+ while (flightStream.next()) {
+
+ }
+ return PutResult.getDefaultInstance();
+ }
+ };
+ }
+
+ @Override
+ public void getStream(Ticket ticket, ServerStreamListener listener) {
+ final int size = 10;
+
+ IntVector iv = new IntVector("c1", allocator);
+ VectorSchemaRoot root = VectorSchemaRoot.of(iv);
+ listener.start(root);
+
+ //batch 1
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+
+ // batch 2
+
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i + size);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+ root.clear();
+ listener.completed();
+ }
+
+ @Override
+ public void close() throws Exception {
+ allocator.close();
+ }
+
+ @Override
+ public FlightInfo getFlightInfo(FlightDescriptor descriptor) {
+ FlightGetInfo getInfo = FlightGetInfo.newBuilder()
+ .setFlightDescriptor(Flight.FlightDescriptor.newBuilder()
+ .setType(DescriptorType.CMD)
+ .setCmd(ByteString.copyFrom("cool thing", Charsets.UTF_8)))
+ .build();
+ return new FlightInfo(getInfo);
+ }
+
+ @Override
+ public Result doAction(Action action) {
+ switch (action.getType()) {
+ case "hello":
+ return new Result("world".getBytes(Charsets.UTF_8));
+ default:
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public void listActions(StreamListener<ActionType> listener) {
+ listener.onNext(new ActionType("get", ""));
+ listener.onNext(new ActionType("put", ""));
+ listener.onNext(new ActionType("hello", ""));
+ listener.onCompleted();
+ }
+
+ }
+
+
+}
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java b/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
new file mode 100644
index 0000000..b8231a9
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/auth/TestAuth.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.auth;
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.arrow.flight.Criteria;
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.NoOpFlightProducer;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import com.google.common.collect.ImmutableList;
+
+public class TestAuth {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private static final String USERNAME = "flight";
+ private static final String PASSWORD = "woohoo";
+ private static final byte[] VALID_TOKEN = "my_token".getBytes();
+
+ private FlightClient client;
+ private FlightServer server;
+ private BufferAllocator allocator;
+
+ @Test
+ public void validAuth() {
+ client.authenticateBasic(USERNAME, PASSWORD);
+ ImmutableList.copyOf(client.listFlights(Criteria.ALL));
+ }
+
+ @Test
+ public void invalidAuth() {
+
+ thrown.expectMessage("PERMISSION_DENIED");
+ client.authenticateBasic(USERNAME, "WRONG");
+
+ thrown.expectMessage("PERMISSION_DENIED");
+ client.listFlights(Criteria.ALL);
+ }
+
+ @Test
+ public void didntAuth() {
+ thrown.expectMessage("PERMISSION_DENIED");
+ client.listFlights(Criteria.ALL);
+ }
+
+ @Before
+ public void setup() throws IOException {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ final Location l = new Location("localhost", 12233);
+
+ final BasicServerAuthHandler.BasicAuthValidator validator = new BasicServerAuthHandler.BasicAuthValidator() {
+
+ @Override
+ public boolean isValid(byte[] token) {
+ return Arrays.equals(token, VALID_TOKEN);
+ }
+
+ @Override
+ public byte[] getToken(String username, String password) throws Exception {
+ if (USERNAME.equals(username) && PASSWORD.equals(password)) {
+ return VALID_TOKEN;
+ } else {
+ throw new IllegalArgumentException("invalid credentials");
+ }
+ }
+ };
+
+ server = new FlightServer(
+ allocator,
+ l.getPort(),
+ new NoOpFlightProducer() {
+ @Override
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener) {
+ listener.onCompleted();
+ }
+ },
+ new BasicServerAuthHandler(validator));
+
+ server.start();
+ client = new FlightClient(allocator, l);
+ }
+
+
+
+ @After
+ public void shutdown() throws Exception {
+ AutoCloseables.close(client, server, allocator);
+ }
+
+}
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/example/TestExampleServer.java b/java/flight/src/test/java/org/apache/arrow/flight/example/TestExampleServer.java
new file mode 100644
index 0000000..f2214b0
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/example/TestExampleServer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.example;
+
+import java.io.IOException;
+
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightClient.ClientStreamListener;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Ensure that example server supports get and put.
+ */
+public class TestExampleServer {
+
+ private BufferAllocator allocator;
+ private BufferAllocator caseAllocator;
+ private ExampleFlightServer server;
+ private FlightClient client;
+
+ @Before
+ public void start() throws IOException {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+
+ Location l = new Location("localhost", 12233);
+ if (!Boolean.getBoolean("disableServer")) {
+ System.out.println("Starting server.");
+ server = new ExampleFlightServer(allocator, l);
+ server.start();
+ } else {
+ System.out.println("Skipping server startup.");
+ }
+ client = new FlightClient(allocator, l);
+ caseAllocator = allocator.newChildAllocator("test-case", 0, Long.MAX_VALUE);
+ }
+
+ @After
+ public void after() throws Exception {
+ AutoCloseables.close(server, client, caseAllocator, allocator);
+ }
+
+ @Test
+ public void putStream() throws Exception {
+ BufferAllocator a = caseAllocator;
+ final int size = 10;
+
+ IntVector iv = new IntVector("c1", a);
+
+ VectorSchemaRoot root = VectorSchemaRoot.of(iv);
+ ClientStreamListener listener = client.startPut(FlightDescriptor.path("hello"), root);
+
+ //batch 1
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+
+ // batch 2
+
+ root.allocateNew();
+ for (int i = 0; i < size; i++) {
+ iv.set(i, i + size);
+ }
+ iv.setValueCount(size);
+ root.setRowCount(size);
+ listener.putNext();
+ root.clear();
+ listener.completed();
+
+ // wait for ack to avoid memory leaks.
+ listener.getResult();
+
+ FlightInfo info = client.getInfo(FlightDescriptor.path("hello"));
+ FlightStream stream = client.getStream(info.getEndpoints().get(0).getTicket());
+ VectorSchemaRoot newRoot = stream.getRoot();
+ while (stream.next()) {
+ newRoot.clear();
+ }
+ }
+}
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java b/java/flight/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java
new file mode 100644
index 0000000..2dd8d84
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/perf/PerformanceTestServer.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.perf;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+
+import org.apache.arrow.flight.Action;
+import org.apache.arrow.flight.ActionType;
+import org.apache.arrow.flight.Criteria;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightEndpoint;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightProducer;
+import org.apache.arrow.flight.FlightServer;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.Result;
+import org.apache.arrow.flight.Ticket;
+import org.apache.arrow.flight.auth.ServerAuthHandler;
+import org.apache.arrow.flight.example.ExampleFlightServer;
+import org.apache.arrow.flight.impl.Flight.PutResult;
+import org.apache.arrow.flight.perf.impl.PerfOuterClass.Perf;
+import org.apache.arrow.flight.perf.impl.PerfOuterClass.Token;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.protobuf.InvalidProtocolBufferException;
+
+public class PerformanceTestServer implements AutoCloseable {
+
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ExampleFlightServer.class);
+
+ private final FlightServer flightServer;
+ private final Location location;
+ private final BufferAllocator allocator;
+ private final PerfProducer producer;
+
+ public PerformanceTestServer(BufferAllocator incomingAllocator, Location location) {
+ this.allocator = incomingAllocator.newChildAllocator("perf-server", 0, Long.MAX_VALUE);
+ this.location = location;
+ this.producer = new PerfProducer();
+ this.flightServer = new FlightServer(this.allocator, location.getPort(), producer, ServerAuthHandler.NO_OP);
+ }
+
+ public Location getLocation() {
+ return location;
+ }
+
+ public void start() throws IOException {
+ flightServer.start();
+ }
+
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(flightServer, allocator);
+ }
+
+ private final class PerfProducer implements FlightProducer {
+
+ @Override
+ public void getStream(Ticket ticket, ServerStreamListener listener) {
+ VectorSchemaRoot root = null;
+ try {
+ Token token = Token.parseFrom(ticket.getBytes());
+ Perf perf = token.getDefinition();
+ Schema schema = Schema.deserialize(ByteBuffer.wrap(perf.getSchema().toByteArray()));
+ root = VectorSchemaRoot.create(schema, allocator);
+ BigIntVector a = (BigIntVector) root.getVector("a");
+ BigIntVector b = (BigIntVector) root.getVector("b");
+ BigIntVector c = (BigIntVector) root.getVector("c");
+ BigIntVector d = (BigIntVector) root.getVector("d");
+ listener.start(root);
+ root.allocateNew();
+
+ int current = 0;
+ long i = token.getStart();
+ while (i < token.getEnd()) {
+ if (listener.isCancelled()) {
+ root.clear();
+ return;
+ }
+
+ if (TestPerf.VALIDATE) {
+ a.setSafe(current, i);
+ }
+
+ i++;
+ current++;
+ if (i % perf.getRecordsPerBatch() == 0) {
+ root.setRowCount(current);
+
+ while (!listener.isReady()) {
+ //Thread.sleep(0, nanos);
+ }
+
+ if (listener.isCancelled()) {
+ root.clear();
+ return;
+ }
+ listener.putNext();
+ current = 0;
+ root.allocateNew();
+ }
+ }
+
+ // send last partial batch.
+ if (current != 0) {
+ root.setRowCount(current);
+ listener.putNext();
+ }
+ listener.completed();
+ } catch (InvalidProtocolBufferException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ AutoCloseables.close(root);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void listFlights(Criteria criteria, StreamListener<FlightInfo> listener) {
+ }
+
+ @Override
+ public FlightInfo getFlightInfo(FlightDescriptor descriptor) {
+ try {
+ Preconditions.checkArgument(descriptor.isCommand());
+ Perf exec = Perf.parseFrom(descriptor.getCommand());
+
+ final Schema pojoSchema = new Schema(ImmutableList.of(
+ Field.nullable("a", MinorType.BIGINT.getType()),
+ Field.nullable("b", MinorType.BIGINT.getType()),
+ Field.nullable("c", MinorType.BIGINT.getType()),
+ Field.nullable("d", MinorType.BIGINT.getType())
+ ));
+
+ Token token = Token.newBuilder().setDefinition(exec)
+ .setStart(0)
+ .setEnd(exec.getRecordsPerStream())
+ .build();
+ final Ticket ticket = new Ticket(token.toByteArray());
+
+ List<FlightEndpoint> endpoints = new ArrayList<>();
+ for (int i = 0; i < exec.getStreamCount(); i++) {
+ endpoints.add(new FlightEndpoint(ticket, getLocation()));
+ }
+
+ return new FlightInfo(pojoSchema, descriptor, endpoints, -1,
+ exec.getRecordsPerStream() * exec.getStreamCount());
+ } catch (InvalidProtocolBufferException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public Callable<PutResult> acceptPut(FlightStream flightStream) {
+ return null;
+ }
+
+ @Override
+ public Result doAction(Action action) {
+ return null;
+ }
+
+ @Override
+ public void listActions(StreamListener<ActionType> listener) {
+ }
+
+ }
+}
+
+
+
diff --git a/java/flight/src/test/java/org/apache/arrow/flight/perf/TestPerf.java b/java/flight/src/test/java/org/apache/arrow/flight/perf/TestPerf.java
new file mode 100644
index 0000000..6a59fe5
--- /dev/null
+++ b/java/flight/src/test/java/org/apache/arrow/flight/perf/TestPerf.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.flight.perf;
+
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.arrow.flight.FlightClient;
+import org.apache.arrow.flight.FlightDescriptor;
+import org.apache.arrow.flight.FlightInfo;
+import org.apache.arrow.flight.FlightStream;
+import org.apache.arrow.flight.Location;
+import org.apache.arrow.flight.Ticket;
+import org.apache.arrow.flight.perf.impl.PerfOuterClass.Perf;
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.VectorSchemaRoot;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
+import org.apache.arrow.vector.types.pojo.Schema;
+import org.junit.Test;
+
+import com.google.common.base.MoreObjects;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.flatbuffers.FlatBufferBuilder;
+import com.google.protobuf.ByteString;
+
+public class TestPerf {
+
+ public static final boolean VALIDATE = false;
+
+ public static FlightDescriptor getPerfFlightDescriptor(long recordCount, int recordsPerBatch, int streamCount) {
+ final Schema pojoSchema = new Schema(ImmutableList.of(
+ Field.nullable("a", MinorType.BIGINT.getType()),
+ Field.nullable("b", MinorType.BIGINT.getType()),
+ Field.nullable("c", MinorType.BIGINT.getType()),
+ Field.nullable("d", MinorType.BIGINT.getType())
+ ));
+
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ pojoSchema.getSchema(builder);
+
+ return FlightDescriptor.command(Perf.newBuilder()
+ .setRecordsPerStream(recordCount)
+ .setRecordsPerBatch(recordsPerBatch)
+ .setSchema(ByteString.copyFrom(pojoSchema.toByteArray()))
+ .setStreamCount(streamCount)
+ .build()
+ .toByteArray());
+ }
+
+ @Test
+ public void throughput() throws Exception {
+ for (int i = 0; i < 10; i++) {
+ final Location l = new Location("localhost", 12233);
+ try (
+ final BufferAllocator a = new RootAllocator(Long.MAX_VALUE);
+ final PerformanceTestServer server = new PerformanceTestServer(a, l);
+ final FlightClient client = new FlightClient(a, l);
+ ) {
+
+ server.start();
+
+ final FlightInfo info = client.getInfo(getPerfFlightDescriptor(50_000_000l, 4095, 2));
+ ListeningExecutorService pool = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(4));
+ List<ListenableFuture<Result>> results = info.getEndpoints()
+ .stream()
+ .map(t -> new Consumer(client, t.getTicket()))
+ .map(t -> pool.submit(t))
+ .collect(Collectors.toList());
+
+ Futures.whenAllSucceed(results);
+ Result r = new Result();
+ for (ListenableFuture<Result> f : results) {
+ r.add(f.get());
+ }
+
+ double seconds = r.nanos * 1.0d / 1000 / 1000 / 1000;
+ System.out.println(String.format(
+ "Transferred %d records totaling %s bytes at %f mb/s. %f record/s. %f batch/s.",
+ r.rows,
+ r.bytes,
+ (r.bytes * 1.0d / 1024 / 1024) / seconds,
+ (r.rows * 1.0d) / seconds,
+ (r.batches * 1.0d) / seconds
+ ));
+ }
+ }
+ }
+
+ private final class Consumer implements Callable<Result> {
+
+ private final FlightClient client;
+ private final Ticket ticket;
+
+ public Consumer(FlightClient client, Ticket ticket) {
+ super();
+ this.client = client;
+ this.ticket = ticket;
+ }
+
+ @Override
+ public Result call() throws Exception {
+ final Result r = new Result();
+ Stopwatch watch = Stopwatch.createStarted();
+ FlightStream stream = client.getStream(ticket);
+ final VectorSchemaRoot root = stream.getRoot();
+ try {
+ BigIntVector a = (BigIntVector) root.getVector("a");
+ while (stream.next()) {
+ int rows = root.getRowCount();
+ long aSum = r.aSum;
+ for (int i = 0; i < rows; i++) {
+ if (VALIDATE) {
+ aSum += a.get(i);
+ }
+ }
+ r.bytes += rows * 32;
+ r.rows += rows;
+ r.aSum = aSum;
+ r.batches++;
+ }
+
+ r.nanos = watch.elapsed(TimeUnit.NANOSECONDS);
+ return r;
+ } finally {
+ root.clear();
+ }
+ }
+
+ }
+
+ private final class Result {
+ private long rows;
+ private long aSum;
+ private long bytes;
+ private long nanos;
+ private long batches;
+
+ public void add(Result r) {
+ rows += r.rows;
+ aSum += r.aSum;
+ bytes += r.bytes;
+ batches += r.batches;
+ nanos = Math.max(nanos, r.nanos);
+ }
+
+ @Override
+ public String toString() {
+ return MoreObjects.toStringHelper(this)
+ .add("rows", rows)
+ .add("aSum", aSum)
+ .add("batches", batches)
+ .add("bytes", bytes)
+ .add("nanos", nanos)
+ .toString();
+ }
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/flight/src/test/protobuf/perf.proto
similarity index 64%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/flight/src/test/protobuf/perf.proto
index a5b5c9d..99f35a9 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/flight/src/test/protobuf/perf.proto
@@ -1,4 +1,4 @@
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,30 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+syntax = "proto3";
-import com.google.common.base.Preconditions;
+option java_package = "org.apache.arrow.flight.perf.impl";
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
-
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
- }
+message Perf {
+ bytes schema = 1;
+ int32 stream_count = 2;
+ int64 records_per_stream = 3;
+ int32 records_per_batch = 4;
+}
- public V getVector() {
- return vector;
- }
+/*
+ * Payload of ticket
+ */
+message Token {
- public boolean isCreated() {
- return created;
- }
+ // definition of entire flight.
+ Perf definition = 1;
+
+ // inclusive start
+ int64 start = 2;
+
+ // exclusive end
+ int64 end = 3;
+
}
+
diff --git a/java/flight/src/test/resources/logback.xml b/java/flight/src/test/resources/logback.xml
new file mode 100644
index 0000000..e409e92
--- /dev/null
+++ b/java/flight/src/test/resources/logback.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
+ license agreements. See the NOTICE file distributed with this work for additional
+ information regarding copyright ownership. The ASF licenses this file to
+ You under the Apache License, Version 2.0 (the "License"); you may not use
+ this file except in compliance with the License. You may obtain a copy of
+ the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
+ by applicable law or agreed to in writing, software distributed under the
+ License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
+ OF ANY KIND, either express or implied. See the License for the specific
+ language governing permissions and limitations under the License. -->
+
+<configuration>
+ <appender name="SOCKET"
+ class="de.huxhorn.lilith.logback.appender.ClassicMultiplexSocketAppender">
+ <Compressing>true</Compressing>
+ <ReconnectionDelay>10000</ReconnectionDelay>
+ <IncludeCallerData>true</IncludeCallerData>
+ <RemoteHosts>${LILITH_HOSTNAME:-localhost}</RemoteHosts>
+ </appender>
+
+ <logger name="org.apache.arrow" additivity="false">
+ <level value="info" />
+ <appender-ref ref="FILE" />
+ </logger>
+
+</configuration>
\ No newline at end of file
diff --git a/java/memory/pom.xml b/java/memory/pom.xml
index 690e408..38f5b9d 100644
--- a/java/memory/pom.xml
+++ b/java/memory/pom.xml
@@ -25,10 +25,6 @@
<artifactId>jsr305</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
diff --git a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
index 5d5d22e..f9a75db 100644
--- a/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
+++ b/java/memory/src/main/java/io/netty/buffer/ArrowBuf.java
@@ -38,8 +38,7 @@ import org.apache.arrow.memory.BoundsChecking;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.BufferManager;
import org.apache.arrow.memory.util.HistoricalLog;
-
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
import io.netty.util.internal.PlatformDependent;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
index 97f8370..791d7c8 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/Accountant.java
@@ -22,7 +22,7 @@ import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.concurrent.ThreadSafe;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
/**
* Provides a concurrent way to manage account for memory usage without locking. Used as basis
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
index d3320bd..4d916a1 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/AllocationManager.java
@@ -29,8 +29,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.arrow.memory.BaseAllocator.Verbosity;
import org.apache.arrow.memory.util.AutoCloseableLock;
import org.apache.arrow.memory.util.HistoricalLog;
-
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
import io.netty.buffer.ArrowBuf;
import io.netty.buffer.PooledByteBufAllocatorL;
@@ -199,7 +198,7 @@ public class AllocationManager {
public class BufferLedger implements ValueWithKeyIncluded<BaseAllocator> {
private final IdentityHashMap<ArrowBuf, Object> buffers =
- BaseAllocator.DEBUG ? new IdentityHashMap<ArrowBuf, Object>() : null;
+ BaseAllocator.DEBUG ? new IdentityHashMap<>() : null;
private final long ledgerId = LEDGER_ID_GENERATOR.incrementAndGet(); // unique ID assigned to
// each ledger
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
index 5003a4e..c94e68f 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/BaseAllocator.java
@@ -26,8 +26,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.arrow.memory.AllocationManager.BufferLedger;
import org.apache.arrow.memory.util.AssertionUtil;
import org.apache.arrow.memory.util.HistoricalLog;
-
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
import io.netty.buffer.ArrowBuf;
import io.netty.buffer.UnsafeDirectLittleEndian;
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
index 311922a..fabca6c 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/LowCostIdentityHashMap.java
@@ -17,8 +17,8 @@
package org.apache.arrow.memory;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.util.VisibleForTesting;
/**
* Highly specialized IdentityHashMap that implements only partial
diff --git a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
index 161b81a..f8a7314 100644
--- a/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
+++ b/java/memory/src/main/java/org/apache/arrow/memory/RootAllocator.java
@@ -18,7 +18,7 @@
package org.apache.arrow.memory;
-import com.google.common.annotations.VisibleForTesting;
+import org.apache.arrow.util.VisibleForTesting;
/**
* The root allocator for using direct memory inside a Drillbit. Supports creating a
diff --git a/java/memory/src/main/java/org/apache/arrow/util/Preconditions.java b/java/memory/src/main/java/org/apache/arrow/util/Preconditions.java
new file mode 100644
index 0000000..c64af2d
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/util/Preconditions.java
@@ -0,0 +1,1306 @@
+/*
+ * Copyright (C) 2007 The Guava Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+ * in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * Lifted from Guava 20.0 to avoid dependency for core Arrow libraries.
+ */
+
+package org.apache.arrow.util;
+
+/**
+ * Static convenience methods that help a method or constructor check whether it was invoked
+ * correctly (whether its <i>preconditions</i> have been met). These methods generally accept a
+ * {@code boolean} expression which is expected to be {@code true} (or in the case of {@code
+ * checkNotNull}, an object reference which is expected to be non-null). When {@code false} (or
+ * {@code null}) is passed instead, the {@code Preconditions} method throws an unchecked exception,
+ * which helps the calling method communicate to <i>its</i> caller that <i>that</i> caller has made
+ * a mistake. Example: <pre> {@code
+ *
+ * /**
+ * * Returns the positive square root of the given value.
+ * *
+ * * @throws IllegalArgumentException if the value is negative
+ * *}{@code /
+ * public static double sqrt(double value) {
+ * Preconditions.checkArgument(value >= 0.0, "negative value: %s", value);
+ * // calculate the square root
+ * }
+ *
+ * void exampleBadCaller() {
+ * double d = sqrt(-1.0);
+ * }}</pre>
+ *
+ * In this example, {@code checkArgument} throws an {@code IllegalArgumentException} to indicate
+ * that {@code exampleBadCaller} made an error in <i>its</i> call to {@code sqrt}.
+ *
+ * <h3>Warning about performance</h3>
+ *
+ * <p>The goal of this class is to improve readability of code, but in some circumstances this may
+ * come at a significant performance cost. Remember that parameter values for message construction
+ * must all be computed eagerly, and autoboxing and varargs array creation may happen as well, even
+ * when the precondition check then succeeds (as it should almost always do in production). In some
+ * circumstances these wasted CPU cycles and allocations can add up to a real problem.
+ * Performance-sensitive precondition checks can always be converted to the customary form:
+ * <pre> {@code
+ *
+ * if (value < 0.0) {
+ * throw new IllegalArgumentException("negative value: " + value);
+ * }}</pre>
+ *
+ * <h3>Other types of preconditions</h3>
+ *
+ * <p>Not every type of precondition failure is supported by these methods. Continue to throw
+ * standard JDK exceptions such as {@link java.util.NoSuchElementException} or
+ * {@link UnsupportedOperationException} in the situations they are intended for.
+ *
+ * <h3>Non-preconditions</h3>
+ *
+ * <p>It is of course possible to use the methods of this class to check for invalid conditions
+ * which are <i>not the caller's fault</i>. Doing so is <b>not recommended</b> because it is
+ * misleading to future readers of the code and of stack traces. See
+ * <a href="https://github.com/google/guava/wiki/ConditionalFailuresExplained">Conditional failures
+ * explained</a> in the Guava User Guide for more advice.
+ *
+ * <h3>{@code java.util.Objects.requireNonNull()}</h3>
+ *
+ * <p>Projects which use {@code com.google.common} should generally avoid the use of
+ * {@link java.util.Objects#requireNonNull(Object)}. Instead, use whichever of
+ * {@link #checkNotNull(Object)} or {@link Verify#verifyNotNull(Object)} is appropriate to the
+ * situation. (The same goes for the message-accepting overloads.)
+ *
+ * <h3>Only {@code %s} is supported</h3>
+ *
+ * <p>In {@code Preconditions} error message template strings, only the {@code "%s"} specifier is
+ * supported, not the full range of {@link java.util.Formatter} specifiers.
+ *
+ * <h3>More information</h3>
+ *
+ * <p>See the Guava User Guide on
+ * <a href="https://github.com/google/guava/wiki/PreconditionsExplained">using {@code
+ * Preconditions}</a>.
+ *
+ * @author Kevin Bourrillion
+ * @since 2.0
+ */
+public final class Preconditions {
+ private Preconditions() {}
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @throws IllegalArgumentException if {@code expression} is false
+ */
+ public static void checkArgument(boolean expression) {
+ if (!expression) {
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @param errorMessage the exception message to use if the check fails; will be converted to a
+ * string using {@link String#valueOf(Object)}
+ * @throws IllegalArgumentException if {@code expression} is false
+ */
+ public static void checkArgument(boolean expression, Object errorMessage) {
+ if (!expression) {
+ throw new IllegalArgumentException(String.valueOf(errorMessage));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @param errorMessageTemplate a template for the exception message should the check fail. The
+ * message is formed by replacing each {@code %s} placeholder in the template with an
+ * argument. These are matched by position - the first {@code %s} gets {@code
+ * errorMessageArgs[0]}, etc. Unmatched arguments will be appended to the formatted message in
+ * square braces. Unmatched placeholders will be left as-is.
+ * @param errorMessageArgs the arguments to be substituted into the message template. Arguments
+ * are converted to strings using {@link String#valueOf(Object)}.
+ * @throws IllegalArgumentException if {@code expression} is false
+ * @throws NullPointerException if the check fails and either {@code errorMessageTemplate} or
+ * {@code errorMessageArgs} is null (don't let this happen)
+ */
+ public static void checkArgument(
+ boolean expression,
+ String errorMessageTemplate,
+ Object... errorMessageArgs) {
+ if (!expression) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(boolean b, String errorMessageTemplate, char p1) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(boolean b, String errorMessageTemplate, int p1) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(boolean b, String errorMessageTemplate, long p1) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, Object p1) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, char p1, char p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, char p1, int p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, char p1, long p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, char p1, Object p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, int p1, char p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, int p1, int p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, int p1, long p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, int p1, Object p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, long p1, char p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, long p1, int p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, long p1, long p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, long p1, Object p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, Object p1, char p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, Object p1, int p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, Object p1, long p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b, String errorMessageTemplate, Object p1, Object p2) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2, p3));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving one or more parameters to the calling method.
+ *
+ * <p>See {@link #checkArgument(boolean, String, Object...)} for details.
+ */
+ public static void checkArgument(
+ boolean b,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3,
+ Object p4) {
+ if (!b) {
+ throw new IllegalArgumentException(format(errorMessageTemplate, p1, p2, p3, p4));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @throws IllegalStateException if {@code expression} is false
+ */
+ public static void checkState(boolean expression) {
+ if (!expression) {
+ throw new IllegalStateException();
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @param errorMessage the exception message to use if the check fails; will be converted to a
+ * string using {@link String#valueOf(Object)}
+ * @throws IllegalStateException if {@code expression} is false
+ */
+ public static void checkState(boolean expression, Object errorMessage) {
+ if (!expression) {
+ throw new IllegalStateException(String.valueOf(errorMessage));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * @param expression a boolean expression
+ * @param errorMessageTemplate a template for the exception message should the check fail. The
+ * message is formed by replacing each {@code %s} placeholder in the template with an
+ * argument. These are matched by position - the first {@code %s} gets {@code
+ * errorMessageArgs[0]}, etc. Unmatched arguments will be appended to the formatted message in
+ * square braces. Unmatched placeholders will be left as-is.
+ * @param errorMessageArgs the arguments to be substituted into the message template. Arguments
+ * are converted to strings using {@link String#valueOf(Object)}.
+ * @throws IllegalStateException if {@code expression} is false
+ * @throws NullPointerException if the check fails and either {@code errorMessageTemplate} or
+ * {@code errorMessageArgs} is null (don't let this happen)
+ */
+ public static void checkState(
+ boolean expression,
+ String errorMessageTemplate,
+ Object... errorMessageArgs) {
+ if (!expression) {
+ throw new IllegalStateException(format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, char p1) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, int p1) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, long p1) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, Object p1) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, char p1, char p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, char p1, int p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, char p1, long p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, char p1, Object p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, int p1, char p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, int p1, int p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, int p1, long p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, int p1, Object p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, long p1, char p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(boolean b, String errorMessageTemplate, long p1, int p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, long p1, long p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, long p1, Object p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, Object p1, char p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, Object p1, int p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, Object p1, long p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b, String errorMessageTemplate, Object p1, Object p2) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2, p3));
+ }
+ }
+
+ /**
+ * Ensures the truth of an expression involving the state of the calling instance, but not
+ * involving any parameters to the calling method.
+ *
+ * <p>See {@link #checkState(boolean, String, Object...)} for details.
+ */
+ public static void checkState(
+ boolean b,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3,
+ Object p4) {
+ if (!b) {
+ throw new IllegalStateException(format(errorMessageTemplate, p1, p2, p3, p4));
+ }
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * @param reference an object reference
+ * @return the non-null reference that was validated
+ * @throws NullPointerException if {@code reference} is null
+ */
+
+ public static <T> T checkNotNull(T reference) {
+ if (reference == null) {
+ throw new NullPointerException();
+ }
+ return reference;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * @param reference an object reference
+ * @param errorMessage the exception message to use if the check fails; will be converted to a
+ * string using {@link String#valueOf(Object)}
+ * @return the non-null reference that was validated
+ * @throws NullPointerException if {@code reference} is null
+ */
+
+ public static <T> T checkNotNull(T reference, Object errorMessage) {
+ if (reference == null) {
+ throw new NullPointerException(String.valueOf(errorMessage));
+ }
+ return reference;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * @param reference an object reference
+ * @param errorMessageTemplate a template for the exception message should the check fail. The
+ * message is formed by replacing each {@code %s} placeholder in the template with an
+ * argument. These are matched by position - the first {@code %s} gets {@code
+ * errorMessageArgs[0]}, etc. Unmatched arguments will be appended to the formatted message in
+ * square braces. Unmatched placeholders will be left as-is.
+ * @param errorMessageArgs the arguments to be substituted into the message template. Arguments
+ * are converted to strings using {@link String#valueOf(Object)}.
+ * @return the non-null reference that was validated
+ * @throws NullPointerException if {@code reference} is null
+ */
+
+ public static <T> T checkNotNull(
+ T reference, String errorMessageTemplate, Object... errorMessageArgs) {
+ if (reference == null) {
+ // If either of these parameters is null, the right thing happens anyway
+ throw new NullPointerException(format(errorMessageTemplate, errorMessageArgs));
+ }
+ return reference;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, char p1) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, int p1) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, long p1) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, Object p1) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, char p1, char p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, char p1, int p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, char p1, long p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, char p1, Object p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, int p1, char p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, int p1, int p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, int p1, long p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, int p1, Object p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, long p1, char p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, long p1, int p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(T obj, String errorMessageTemplate, long p1, long p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, long p1, Object p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, Object p1, char p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, Object p1, int p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, Object p1, long p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj, String errorMessageTemplate, Object p1, Object p2) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2, p3));
+ }
+ return obj;
+ }
+
+ /**
+ * Ensures that an object reference passed as a parameter to the calling method is not null.
+ *
+ * <p>See {@link #checkNotNull(Object, String, Object...)} for details.
+ */
+
+ public static <T> T checkNotNull(
+ T obj,
+ String errorMessageTemplate,
+ Object p1,
+ Object p2,
+ Object p3,
+ Object p4) {
+ if (obj == null) {
+ throw new NullPointerException(format(errorMessageTemplate, p1, p2, p3, p4));
+ }
+ return obj;
+ }
+
+ /*
+ * All recent hotspots (as of 2009) *really* like to have the natural code
+ *
+ * if (guardExpression) {
+ * throw new BadException(messageExpression);
+ * }
+ *
+ * refactored so that messageExpression is moved to a separate String-returning method.
+ *
+ * if (guardExpression) {
+ * throw new BadException(badMsg(...));
+ * }
+ *
+ * The alternative natural refactorings into void or Exception-returning methods are much slower.
+ * This is a big deal - we're talking factors of 2-8 in microbenchmarks, not just 10-20%. (This is
+ * a hotspot optimizer bug, which should be fixed, but that's a separate, big project).
+ *
+ * The coding pattern above is heavily used in java.util, e.g. in ArrayList. There is a
+ * RangeCheckMicroBenchmark in the JDK that was used to test this.
+ *
+ * But the methods in this class want to throw different exceptions, depending on the args, so it
+ * appears that this pattern is not directly applicable. But we can use the ridiculous, devious
+ * trick of throwing an exception in the middle of the construction of another exception. Hotspot
+ * is fine with that.
+ */
+
+ /**
+ * Ensures that {@code index} specifies a valid <i>element</i> in an array, list or string of size
+ * {@code size}. An element index may range from zero, inclusive, to {@code size}, exclusive.
+ *
+ * @param index a user-supplied index identifying an element of an array, list or string
+ * @param size the size of that array, list or string
+ * @return the value of {@code index}
+ * @throws IndexOutOfBoundsException if {@code index} is negative or is not less than {@code size}
+ * @throws IllegalArgumentException if {@code size} is negative
+ */
+
+ public static int checkElementIndex(int index, int size) {
+ return checkElementIndex(index, size, "index");
+ }
+
+ /**
+ * Ensures that {@code index} specifies a valid <i>element</i> in an array, list or string of size
+ * {@code size}. An element index may range from zero, inclusive, to {@code size}, exclusive.
+ *
+ * @param index a user-supplied index identifying an element of an array, list or string
+ * @param size the size of that array, list or string
+ * @param desc the text to use to describe this index in an error message
+ * @return the value of {@code index}
+ * @throws IndexOutOfBoundsException if {@code index} is negative or is not less than {@code size}
+ * @throws IllegalArgumentException if {@code size} is negative
+ */
+
+ public static int checkElementIndex(int index, int size, String desc) {
+ // Carefully optimized for execution by hotspot (explanatory comment above)
+ if (index < 0 || index >= size) {
+ throw new IndexOutOfBoundsException(badElementIndex(index, size, desc));
+ }
+ return index;
+ }
+
+ private static String badElementIndex(int index, int size, String desc) {
+ if (index < 0) {
+ return format("%s (%s) must not be negative", desc, index);
+ } else if (size < 0) {
+ throw new IllegalArgumentException("negative size: " + size);
+ } else { // index >= size
+ return format("%s (%s) must be less than size (%s)", desc, index, size);
+ }
+ }
+
+ /**
+ * Ensures that {@code index} specifies a valid <i>position</i> in an array, list or string of
+ * size {@code size}. A position index may range from zero to {@code size}, inclusive.
+ *
+ * @param index a user-supplied index identifying a position in an array, list or string
+ * @param size the size of that array, list or string
+ * @return the value of {@code index}
+ * @throws IndexOutOfBoundsException if {@code index} is negative or is greater than {@code size}
+ * @throws IllegalArgumentException if {@code size} is negative
+ */
+
+ public static int checkPositionIndex(int index, int size) {
+ return checkPositionIndex(index, size, "index");
+ }
+
+ /**
+ * Ensures that {@code index} specifies a valid <i>position</i> in an array, list or string of
+ * size {@code size}. A position index may range from zero to {@code size}, inclusive.
+ *
+ * @param index a user-supplied index identifying a position in an array, list or string
+ * @param size the size of that array, list or string
+ * @param desc the text to use to describe this index in an error message
+ * @return the value of {@code index}
+ * @throws IndexOutOfBoundsException if {@code index} is negative or is greater than {@code size}
+ * @throws IllegalArgumentException if {@code size} is negative
+ */
+
+ public static int checkPositionIndex(int index, int size, String desc) {
+ // Carefully optimized for execution by hotspot (explanatory comment above)
+ if (index < 0 || index > size) {
+ throw new IndexOutOfBoundsException(badPositionIndex(index, size, desc));
+ }
+ return index;
+ }
+
+ private static String badPositionIndex(int index, int size, String desc) {
+ if (index < 0) {
+ return format("%s (%s) must not be negative", desc, index);
+ } else if (size < 0) {
+ throw new IllegalArgumentException("negative size: " + size);
+ } else { // index > size
+ return format("%s (%s) must not be greater than size (%s)", desc, index, size);
+ }
+ }
+
+ /**
+ * Ensures that {@code start} and {@code end} specify a valid <i>positions</i> in an array, list
+ * or string of size {@code size}, and are in order. A position index may range from zero to
+ * {@code size}, inclusive.
+ *
+ * @param start a user-supplied index identifying a starting position in an array, list or string
+ * @param end a user-supplied index identifying a ending position in an array, list or string
+ * @param size the size of that array, list or string
+ * @throws IndexOutOfBoundsException if either index is negative or is greater than {@code size},
+ * or if {@code end} is less than {@code start}
+ * @throws IllegalArgumentException if {@code size} is negative
+ */
+ public static void checkPositionIndexes(int start, int end, int size) {
+ // Carefully optimized for execution by hotspot (explanatory comment above)
+ if (start < 0 || end < start || end > size) {
+ throw new IndexOutOfBoundsException(badPositionIndexes(start, end, size));
+ }
+ }
+
+ private static String badPositionIndexes(int start, int end, int size) {
+ if (start < 0 || start > size) {
+ return badPositionIndex(start, size, "start index");
+ }
+ if (end < 0 || end > size) {
+ return badPositionIndex(end, size, "end index");
+ }
+ // end < start
+ return format("end index (%s) must not be less than start index (%s)", end, start);
+ }
+
+ /**
+ * Substitutes each {@code %s} in {@code template} with an argument. These are matched by
+ * position: the first {@code %s} gets {@code args[0]}, etc. If there are more arguments than
+ * placeholders, the unmatched arguments will be appended to the end of the formatted message in
+ * square braces.
+ *
+ * @param template a non-null string containing 0 or more {@code %s} placeholders.
+ * @param args the arguments to be substituted into the message template. Arguments are converted
+ * to strings using {@link String#valueOf(Object)}. Arguments can be null.
+ */
+ // Note that this is somewhat-improperly used from Verify.java as well.
+ static String format(String template, Object... args) {
+ template = String.valueOf(template); // null -> "null"
+
+ // start substituting the arguments into the '%s' placeholders
+ StringBuilder builder = new StringBuilder(template.length() + 16 * args.length);
+ int templateStart = 0;
+ int i = 0;
+ while (i < args.length) {
+ int placeholderStart = template.indexOf("%s", templateStart);
+ if (placeholderStart == -1) {
+ break;
+ }
+ builder.append(template, templateStart, placeholderStart);
+ builder.append(args[i++]);
+ templateStart = placeholderStart + 2;
+ }
+ builder.append(template, templateStart, template.length());
+
+ // if we run out of placeholders, append the extra args in square braces
+ if (i < args.length) {
+ builder.append(" [");
+ builder.append(args[i++]);
+ while (i < args.length) {
+ builder.append(", ");
+ builder.append(args[i++]);
+ }
+ builder.append(']');
+ }
+
+ return builder.toString();
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/memory/src/main/java/org/apache/arrow/util/VisibleForTesting.java
similarity index 64%
copy from java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
copy to java/memory/src/main/java/org/apache/arrow/util/VisibleForTesting.java
index a5b5c9d..e5c9fe6 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/memory/src/main/java/org/apache/arrow/util/VisibleForTesting.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ * <p>
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -16,24 +16,7 @@
* limitations under the License.
*/
-package org.apache.arrow.vector;
+package org.apache.arrow.util;
-import com.google.common.base.Preconditions;
-
-public class AddOrGetResult<V extends ValueVector> {
- private final V vector;
- private final boolean created;
-
- public AddOrGetResult(V vector, boolean created) {
- this.vector = Preconditions.checkNotNull(vector);
- this.created = created;
- }
-
- public V getVector() {
- return vector;
- }
-
- public boolean isCreated() {
- return created;
- }
+public @interface VisibleForTesting {
}
diff --git a/java/pom.xml b/java/pom.xml
index 5c47fac..09c3cdc 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -31,8 +31,8 @@
<target.gen.source.path>${project.basedir}/target/generated-sources</target.gen.source.path>
<dep.junit.version>4.11</dep.junit.version>
<dep.slf4j.version>1.7.25</dep.slf4j.version>
- <dep.guava.version>18.0</dep.guava.version>
- <dep.netty.version>4.1.17.Final</dep.netty.version>
+ <dep.guava.version>20.0</dep.guava.version>
+ <dep.netty.version>4.1.22.Final</dep.netty.version>
<dep.jackson.version>2.7.9</dep.jackson.version>
<dep.hadoop.version>2.7.1</dep.hadoop.version>
<fbs.version>1.2.0-3f79e055</fbs.version>
@@ -97,6 +97,7 @@
<configuration>
<excludeSubProjects>false</excludeSubProjects>
<excludes>
+ <exclude>**/dependency-reduced-pom.xml</exclude>
<exclude>**/*.log</exclude>
<exclude>**/*.css</exclude>
<exclude>**/*.js</exclude>
@@ -627,6 +628,7 @@
<module>tools</module>
<module>adapter/jdbc</module>
<module>plasma</module>
+ <module>flight</module>
</modules>
<profiles>
diff --git a/java/vector/pom.xml b/java/vector/pom.xml
index c6f6a70..b4c15c2 100644
--- a/java/vector/pom.xml
+++ b/java/vector/pom.xml
@@ -59,10 +59,6 @@
<version>1.10</version>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </dependency>
- <dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
</dependency>
@@ -75,10 +71,6 @@
<artifactId>flatbuffers</artifactId>
</dependency>
<dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </dependency>
- <dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
diff --git a/java/vector/src/main/codegen/includes/vv_imports.ftl b/java/vector/src/main/codegen/includes/vv_imports.ftl
index fb733fb..738fef5 100644
--- a/java/vector/src/main/codegen/includes/vv_imports.ftl
+++ b/java/vector/src/main/codegen/includes/vv_imports.ftl
@@ -9,20 +9,15 @@
OF ANY KIND, either express or implied. See the License for the specific
language governing permissions and limitations under the License. -->
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.ObjectArrays;
-import com.google.common.base.Charsets;
-import com.google.common.collect.ObjectArrays;
+import static org.apache.arrow.util.Preconditions.checkArgument;
+import static org.apache.arrow.util.Preconditions.checkState;
import com.google.flatbuffers.FlatBufferBuilder;
-import com.google.common.base.Preconditions;
import io.netty.buffer.*;
import org.apache.arrow.memory.*;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.*;
import org.apache.arrow.vector.types.pojo.*;
diff --git a/java/vector/src/main/codegen/templates/StructWriters.java b/java/vector/src/main/codegen/templates/StructWriters.java
index 0bd2e96..0dc62d1 100644
--- a/java/vector/src/main/codegen/templates/StructWriters.java
+++ b/java/vector/src/main/codegen/templates/StructWriters.java
@@ -32,14 +32,13 @@ package org.apache.arrow.vector.complex.impl;
<#include "/@includes/vv_imports.ftl" />
import java.util.Map;
+import java.util.HashMap;
import org.apache.arrow.vector.holders.RepeatedStructHolder;
import org.apache.arrow.vector.AllocationHelper;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.FieldWriter;
-import com.google.common.collect.Maps;
-
/*
* This class is generated using FreeMarker and the ${.template_name} template.
*/
@@ -48,7 +47,7 @@ public class ${mode}StructWriter extends AbstractFieldWriter {
protected final ${containerClass} container;
private int initialCapacity;
- private final Map<String, FieldWriter> fields = Maps.newHashMap();
+ private final Map<String, FieldWriter> fields = new HashMap<>();
public ${mode}StructWriter(${containerClass} container) {
<#if mode == "Single">
if (container instanceof StructVector) {
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 8b27f39..84a3a30 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -24,7 +24,6 @@
package org.apache.arrow.vector.complex;
<#include "/@includes/vv_imports.ftl" />
-import com.google.common.collect.ImmutableList;
import io.netty.buffer.ArrowBuf;
import java.util.ArrayList;
import java.util.Collections;
@@ -460,11 +459,11 @@ public class UnionVector implements FieldVector {
@Override
public ArrowBuf[] getBuffers(boolean clear) {
- ImmutableList.Builder<ArrowBuf> builder = ImmutableList.builder();
+ List<ArrowBuf> list = new java.util.ArrayList<>();
setReaderAndWriterIndex();
if (getBufferSize() != 0) {
- builder.add(typeBuffer);
- builder.add(internalStruct.getBuffers(clear));
+ list.add(typeBuffer);
+ list.addAll(java.util.Arrays.asList(internalStruct.getBuffers(clear)));
}
if (clear) {
valueCount = 0;
@@ -472,13 +471,12 @@ public class UnionVector implements FieldVector {
typeBuffer.release();
typeBuffer = allocator.getEmpty();
}
- List<ArrowBuf> list = builder.build();
return list.toArray(new ArrowBuf[list.size()]);
}
@Override
public Iterator<ValueVector> iterator() {
- List<ValueVector> vectors = Lists.newArrayList(internalStruct.iterator());
+ List<ValueVector> vectors = org.apache.arrow.util.Collections2.toList(internalStruct.iterator());
return vectors.iterator();
}
diff --git a/java/vector/src/main/codegen/templates/UnionWriter.java b/java/vector/src/main/codegen/templates/UnionWriter.java
index 1915c87..74e050e 100644
--- a/java/vector/src/main/codegen/templates/UnionWriter.java
+++ b/java/vector/src/main/codegen/templates/UnionWriter.java
@@ -39,7 +39,7 @@ public class UnionWriter extends AbstractFieldWriter implements FieldWriter {
UnionVector data;
private StructWriter structWriter;
private UnionListWriter listWriter;
- private List<BaseWriter> writers = Lists.newArrayList();
+ private List<BaseWriter> writers = new java.util.ArrayList<>();
private final NullableStructWriterFactory nullableStructWriterFactory;
public UnionWriter(UnionVector vector) {
diff --git a/java/vector/src/main/java/org/apache/arrow/util/AutoCloseables.java b/java/vector/src/main/java/org/apache/arrow/util/AutoCloseables.java
new file mode 100644
index 0000000..dbfbf16
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/util/AutoCloseables.java
@@ -0,0 +1,205 @@
+/*
+ * Portions Copyright (C) 2017-2018 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * Utilities for AutoCloseable classes.
+ */
+public final class AutoCloseables {
+ // Utility class. Should not be instantiated
+ private AutoCloseables() {
+ }
+
+ public static AutoCloseable all(final Collection<? extends AutoCloseable> autoCloseables) {
+ return new AutoCloseable() {
+ @Override
+ public void close() throws Exception {
+ AutoCloseables.close(autoCloseables);
+ }
+ };
+ }
+
+ /**
+ * Closes all autoCloseables if not null and suppresses exceptions by adding them to t.
+ * @param t the throwable to add suppressed exception to
+ * @param autoCloseables the closeables to close
+ */
+ public static void close(Throwable t, AutoCloseable... autoCloseables) {
+ close(t, Arrays.asList(autoCloseables));
+ }
+
+ /**
+ * Closes all autoCloseables if not null and suppresses exceptions by adding them to t.
+ * @param t the throwable to add suppressed exception to
+ * @param autoCloseables the closeables to close
+ */
+ public static void close(Throwable t, Iterable<? extends AutoCloseable> autoCloseables) {
+ try {
+ close(autoCloseables);
+ } catch (Exception e) {
+ t.addSuppressed(e);
+ }
+ }
+
+ /**
+ * Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one.
+ * @param autoCloseables the closeables to close
+ */
+ public static void close(AutoCloseable... autoCloseables) throws Exception {
+ close(Arrays.asList(autoCloseables));
+ }
+
+ /**
+ * Closes all autoCloseables if not null and suppresses subsequent exceptions if more than one.
+ * @param ac the closeables to close
+ */
+ public static void close(Iterable<? extends AutoCloseable> ac) throws Exception {
+ // this method can be called on a single object if it implements Iterable<AutoCloseable>
+ // like for example VectorContainer make sure we handle that properly
+ if (ac == null) {
+ return;
+ } else if (ac instanceof AutoCloseable) {
+ ((AutoCloseable) ac).close();
+ return;
+ }
+
+ Exception topLevelException = null;
+ for (AutoCloseable closeable : ac) {
+ try {
+ if (closeable != null) {
+ closeable.close();
+ }
+ } catch (Exception e) {
+ if (topLevelException == null) {
+ topLevelException = e;
+ } else if (e != topLevelException) {
+ topLevelException.addSuppressed(e);
+ }
+ }
+ }
+ if (topLevelException != null) {
+ throw topLevelException;
+ }
+ }
+
+ @SafeVarargs
+ public static void close(Iterable<? extends AutoCloseable>...closeables) throws Exception {
+ close(Arrays.asList(closeables).stream()
+ .flatMap((Iterable<? extends AutoCloseable> t) -> Collections2.toList(t).stream())
+ .collect(Collectors.toList()));
+ }
+
+ public static Iterable<AutoCloseable> iter(AutoCloseable... ac) {
+ if (ac.length == 0) {
+ return Collections.emptyList();
+ } else {
+ final List<AutoCloseable> nonNullAc = new ArrayList<>();
+ for (AutoCloseable autoCloseable : ac) {
+ if (autoCloseable != null) {
+ nonNullAc.add(autoCloseable);
+ }
+ }
+ return nonNullAc;
+ }
+ }
+
+ /**
+ * A closeable wrapper that will close the underlying closeables if a commit does not occur.
+ */
+ public static class RollbackCloseable implements AutoCloseable {
+
+ private boolean commit = false;
+ private List<AutoCloseable> closeables;
+
+ public RollbackCloseable(AutoCloseable... closeables) {
+ this.closeables = new ArrayList<>(Arrays.asList(closeables));
+ }
+
+ public <T extends AutoCloseable> T add(T t) {
+ closeables.add(t);
+ return t;
+ }
+
+ public void addAll(AutoCloseable... list) {
+ closeables.addAll(Arrays.asList(list));
+ }
+
+ public void addAll(Iterable<? extends AutoCloseable> list) {
+ for (AutoCloseable ac : list) {
+ closeables.add(ac);
+ }
+ }
+
+ public void commit() {
+ commit = true;
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (!commit) {
+ AutoCloseables.close(closeables);
+ }
+ }
+
+ }
+
+ public static RollbackCloseable rollbackable(AutoCloseable... closeables) {
+ return new RollbackCloseable(closeables);
+ }
+
+ /**
+ * close() an {@see java.lang.AutoCloseable} without throwing a (checked)
+ * {@see java.lang.Exception}. This wraps the close() call with a
+ * try-catch that will rethrow an Exception wrapped with a
+ * {@see java.lang.RuntimeException}, providing a way to call close()
+ * without having to do the try-catch everywhere or propagate the Exception.
+ *
+ * @param autoCloseable the AutoCloseable to close; may be null
+ * @throws RuntimeException if an Exception occurs; the Exception is
+ * wrapped by the RuntimeException
+ */
+ public static void closeNoChecked(final AutoCloseable autoCloseable) {
+ if (autoCloseable != null) {
+ try {
+ autoCloseable.close();
+ } catch (final Exception e) {
+ throw new RuntimeException("Exception while closing: " + e.getMessage(), e);
+ }
+ }
+ }
+
+ private static final AutoCloseable noOpAutocloseable = new AutoCloseable() {
+ @Override
+ public void close() {
+ }
+ };
+
+ /**
+ * @return A do-nothing autocloseable
+ */
+ public static AutoCloseable noop() {
+ return noOpAutocloseable;
+ }
+}
+
diff --git a/java/vector/src/main/java/org/apache/arrow/util/Collections2.java b/java/vector/src/main/java/org/apache/arrow/util/Collections2.java
new file mode 100644
index 0000000..0bbb50f
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/util/Collections2.java
@@ -0,0 +1,54 @@
+/*
+ * Portions Copyright (C) 2017-2018 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.util;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+public class Collections2 {
+
+ public static <T> List<T> toList(Iterator<T> iterator) {
+ List<T> target = new ArrayList<>();
+ iterator.forEachRemaining(target::add);
+ return target;
+ }
+
+ public static <T> List<T> toList(Iterable<T> iterable) {
+ return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList());
+ }
+
+ public static <K,V> Map<K, V> immutableMapCopy(Map<K, V> map) {
+ Map<K,V> newMap = new HashMap<>();
+ newMap.putAll(map);
+ return java.util.Collections.unmodifiableMap(newMap);
+ }
+
+ public static <V> List<V> immutableListCopy(List<V> list) {
+ return list.stream().collect(Collectors.toList());
+ }
+
+ public static <V> List<V> asImmutableList(V...values) {
+ return Collections.unmodifiableList(Arrays.asList(values));
+ }
+}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
index a5b5c9d..4018a73 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/AddOrGetResult.java
@@ -18,7 +18,7 @@
package org.apache.arrow.vector;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
public class AddOrGetResult<V extends ValueVector> {
private final V vector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index bf11da6..74a0767 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -22,12 +22,11 @@ import java.util.Collections;
import java.util.Iterator;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.util.TransferPair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Preconditions;
-
import io.netty.buffer.ArrowBuf;
public abstract class BaseValueVector implements ValueVector {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
index 0faab7b..5044ffc 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BufferLayout.java
@@ -18,7 +18,7 @@
package org.apache.arrow.vector;
-import com.google.common.base.Preconditions;
+import org.apache.arrow.util.Preconditions;
public class BufferLayout {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
index cccb3b2..2359ded 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/TypeLayout.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.BufferLayout.BufferType;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeVisitor;
@@ -43,8 +44,6 @@ import org.apache.arrow.vector.types.pojo.ArrowType.Timestamp;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
-import com.google.common.base.Preconditions;
-
/**
* The buffer layout of vectors for a given type
* It defines its own buffers followed by the buffers for the children
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
index 2ba1adc..800146a 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorLoader.java
@@ -18,18 +18,17 @@
package org.apache.arrow.vector;
-import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.arrow.util.Preconditions.checkArgument;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.arrow.util.Collections2;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.apache.arrow.vector.types.pojo.Field;
-import com.google.common.collect.Iterators;
-
import io.netty.buffer.ArrowBuf;
/**
@@ -63,7 +62,7 @@ public class VectorLoader {
root.setRowCount(recordBatch.getLength());
if (nodes.hasNext() || buffers.hasNext()) {
throw new IllegalArgumentException("not all nodes and buffers were consumed. nodes: " +
- Iterators.toString(nodes) + " buffers: " + Iterators.toString(buffers));
+ Collections2.toList(nodes).toString() + " buffers: " + Collections2.toList(buffers).toString());
}
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
index 3fd33d6..f94de61 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorSchemaRoot.java
@@ -19,11 +19,15 @@
package org.apache.arrow.vector;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.AutoCloseables;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
@@ -37,6 +41,15 @@ public class VectorSchemaRoot implements AutoCloseable {
private final List<FieldVector> fieldVectors;
private final Map<String, FieldVector> fieldVectorsMap = new HashMap<>();
+
+ public VectorSchemaRoot(Iterable<FieldVector> vectors) {
+ this(
+ StreamSupport.stream(vectors.spliterator(), false).map(t -> t.getField()).collect(Collectors.toList()),
+ StreamSupport.stream(vectors.spliterator(), false).collect(Collectors.toList()),
+ 0
+ );
+ }
+
public VectorSchemaRoot(FieldVector parent) {
this(parent.getField().getChildren(), parent.getChildrenFromFields(), parent.getValueCount());
}
@@ -73,8 +86,31 @@ public class VectorSchemaRoot implements AutoCloseable {
return new VectorSchemaRoot(schema, fieldVectors, 0);
}
+ public static VectorSchemaRoot of(FieldVector... vectors) {
+ return new VectorSchemaRoot(Arrays.stream(vectors).collect(Collectors.toList()));
+ }
+
+ /**
+ * Do an adaptive allocation of each vector for memory purposes. Sizes will be based on previously
+ * defined initial allocation for each vector (and subsequent size learnings).
+ */
+ public void allocateNew() {
+ for (FieldVector v : fieldVectors) {
+ v.allocateNew();
+ }
+ }
+
+ /**
+ * Release all the memory for each vector held in this root. This DOES NOT remove vectors from the container.
+ */
+ public void clear() {
+ for (FieldVector v : fieldVectors) {
+ v.clear();
+ }
+ }
+
public List<FieldVector> getFieldVectors() {
- return fieldVectors;
+ return fieldVectors.stream().collect(Collectors.toList());
}
public FieldVector getVector(String name) {
@@ -89,34 +125,30 @@ public class VectorSchemaRoot implements AutoCloseable {
return rowCount;
}
+ /**
+ * Set the row count of all the vectors in this container. Also sets the value
+ * count for each root level contained FieldVector.
+ * @param rowCount Number of records.
+ */
public void setRowCount(int rowCount) {
this.rowCount = rowCount;
+ for (FieldVector v : getFieldVectors()) {
+ v.setValueCount(rowCount);
+ }
}
@Override
public void close() {
- RuntimeException ex = null;
- for (FieldVector fieldVector : fieldVectors) {
- try {
- fieldVector.close();
- } catch (RuntimeException e) {
- ex = chain(ex, e);
- }
- }
- if (ex != null) {
+ try {
+ AutoCloseables.close(fieldVectors);
+ } catch (RuntimeException ex) {
throw ex;
+ } catch (Exception ex) {
+ // should never happen since FieldVector.close() doesn't throw IOException
+ throw new RuntimeException(ex);
}
}
- private RuntimeException chain(RuntimeException root, RuntimeException e) {
- if (root == null) {
- root = e;
- } else {
- root.addSuppressed(e);
- }
- return root;
- }
-
private void printRow(StringBuilder sb, List<Object> row) {
boolean first = true;
for (Object v : row) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
index e38492e..8647395 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractStructVector.java
@@ -22,18 +22,16 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.MapWithOrdinal;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-
import io.netty.buffer.ArrowBuf;
/*
@@ -233,11 +231,9 @@ public abstract class AbstractStructVector extends AbstractContainerVector {
}
protected List<String> getChildFieldNames() {
- ImmutableList.Builder<String> builder = ImmutableList.builder();
- for (ValueVector child : getChildren()) {
- builder.add(child.getField().getName());
- }
- return builder.build();
+ return getChildren().stream()
+ .map(child -> child.getField().getName())
+ .collect(Collectors.toList());
}
/**
@@ -257,7 +253,7 @@ public abstract class AbstractStructVector extends AbstractContainerVector {
* @return a list of scalar child vectors recursing the entire vector hierarchy.
*/
public List<ValueVector> getPrimitiveVectors() {
- final List<ValueVector> primitiveVectors = Lists.newArrayList();
+ final List<ValueVector> primitiveVectors = new ArrayList<>();
for (final ValueVector v : vectors.values()) {
if (v instanceof AbstractStructVector) {
AbstractStructVector structVector = (AbstractStructVector) v;
@@ -285,7 +281,7 @@ public abstract class AbstractStructVector extends AbstractContainerVector {
@Override
public ArrowBuf[] getBuffers(boolean clear) {
- final List<ArrowBuf> buffers = Lists.newArrayList();
+ final List<ArrowBuf> buffers = new ArrayList<>();
for (final ValueVector vector : vectors.values()) {
for (final ArrowBuf buf : vector.getBuffers(false)) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 21ab3ad..9d40188 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -18,11 +18,15 @@
package org.apache.arrow.vector.complex;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseFixedWidthVector;
import org.apache.arrow.vector.BaseValueVector;
@@ -38,9 +42,6 @@ import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
import io.netty.buffer.ArrowBuf;
public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
@@ -240,8 +241,10 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
if (getBufferSize() == 0) {
buffers = new ArrowBuf[0];
} else {
- buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer}, vector.getBuffers(false),
- ArrowBuf.class);
+ List<ArrowBuf> list = new ArrayList<>();
+ list.add(offsetBuffer);
+ list.addAll(Arrays.asList(vector.getBuffers(false)));
+ buffers = list.toArray(new ArrowBuf[list.size()]);
}
if (clear) {
for (ArrowBuf buffer : buffers) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
index cd3a2ae..74e1caa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -18,10 +18,9 @@
package org.apache.arrow.vector.complex;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.UInt4Vector;
-import com.google.common.base.Preconditions;
-
/**
* A helper class that is used to track and populate empty values in repeated value vectors.
*/
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
index 7c77ee9..55e5501 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/FixedSizeListVector.java
@@ -21,12 +21,24 @@ package org.apache.arrow.vector.complex;
import static java.util.Collections.singletonList;
import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.util.Preconditions;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.impl.UnionFixedSizeListReader;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
import org.apache.arrow.vector.types.Types.MinorType;
@@ -34,11 +46,11 @@ import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
-import org.apache.arrow.vector.util.*;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.OversizedAllocationException;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+import org.apache.arrow.vector.util.TransferPair;
import io.netty.buffer.ArrowBuf;
@@ -85,7 +97,7 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
@Override
public Field getField() {
- List<Field> children = ImmutableList.of(getDataVector().getField());
+ List<Field> children = Collections.singletonList(getDataVector().getField());
return new Field(name, fieldType, children);
}
@@ -287,8 +299,10 @@ public class FixedSizeListVector extends BaseValueVector implements FieldVector,
if (getBufferSize() == 0) {
buffers = new ArrowBuf[0];
} else {
- buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, vector.getBuffers(false),
- ArrowBuf.class);
+ List<ArrowBuf> list = new ArrayList<>();
+ list.add(validityBuffer);
+ list.addAll(Arrays.asList(vector.getBuffers(false)));
+ buffers = list.toArray(new ArrowBuf[list.size()]);
}
if (clear) {
for (ArrowBuf buffer : buffers) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index cc96edb..405989d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,10 +18,12 @@
package org.apache.arrow.vector.complex;
-import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singletonList;
+import static org.apache.arrow.util.Preconditions.checkNotNull;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import org.apache.arrow.memory.BaseAllocator;
@@ -49,9 +51,6 @@ import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ObjectArrays;
-
import io.netty.buffer.ArrowBuf;
public class ListVector extends BaseRepeatedValueVector implements FieldVector, PromotableVector {
@@ -548,7 +547,7 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
@Override
public Field getField() {
- return new Field(name, fieldType, ImmutableList.of(getDataVector().getField()));
+ return new Field(name, fieldType, Collections.singletonList(getDataVector().getField()));
}
@Override
@@ -588,9 +587,11 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector,
if (getBufferSize() == 0) {
buffers = new ArrowBuf[0];
} else {
- buffers = ObjectArrays.concat(new ArrowBuf[]{offsetBuffer},
- ObjectArrays.concat(new ArrowBuf[]{validityBuffer},
- vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+ List<ArrowBuf> list = new ArrayList<>();
+ list.add(offsetBuffer);
+ list.add(validityBuffer);
+ list.addAll(Arrays.asList(vector.getBuffers(false)));
+ buffers = list.toArray(new ArrowBuf[list.size()]);
}
if (clear) {
for (ArrowBuf buffer : buffers) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
index b18767d..2b344e4 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NonNullableStructVector.java
@@ -18,7 +18,7 @@
package org.apache.arrow.vector.complex;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.arrow.util.Preconditions.checkNotNull;
import java.util.ArrayList;
import java.util.Collection;
@@ -26,8 +26,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import javax.annotation.Nullable;
-
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.*;
import org.apache.arrow.vector.complex.impl.SingleStructReaderImpl;
@@ -41,9 +39,6 @@ import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.Ordering;
-import com.google.common.primitives.Ints;
-
import io.netty.buffer.ArrowBuf;
public class NonNullableStructVector extends AbstractStructVector {
@@ -248,17 +243,10 @@ public class NonNullableStructVector extends AbstractStructVector {
return 0;
}
- final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
- @Override
- public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
- return Ints.compare(
- checkNotNull(left).getValueCapacity(),
- checkNotNull(right).getValueCapacity()
- );
- }
- };
-
- return natural.min(getChildren()).getValueCapacity();
+ return getChildren().stream()
+ .mapToInt(child -> child.getValueCapacity())
+ .min()
+ .getAsInt();
}
@Override
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
index 258f4a1..258adaa 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/StructVector.java
@@ -18,14 +18,19 @@
package org.apache.arrow.vector.complex;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.arrow.util.Preconditions.checkNotNull;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.BitVectorHelper;
+import org.apache.arrow.vector.BufferBacked;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.complex.impl.NullableStructReaderImpl;
import org.apache.arrow.vector.complex.impl.NullableStructWriter;
import org.apache.arrow.vector.holders.ComplexHolder;
@@ -39,8 +44,6 @@ import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.OversizedAllocationException;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.ObjectArrays;
-
import io.netty.buffer.ArrowBuf;
public class StructVector extends NonNullableStructVector implements FieldVector {
@@ -274,8 +277,10 @@ public class StructVector extends NonNullableStructVector implements FieldVector
if (getBufferSize() == 0) {
buffers = new ArrowBuf[0];
} else {
- buffers = ObjectArrays.concat(new ArrowBuf[]{validityBuffer}, super.getBuffers(false),
- ArrowBuf.class);
+ List<ArrowBuf> list = new ArrayList<>();
+ list.add(validityBuffer);
+ list.addAll(Arrays.asList(super.getBuffers(false)));
+ buffers = list.toArray(new ArrowBuf[list.size()]);
}
if (clear) {
for (ArrowBuf buffer : buffers) {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index b246599..3897b93 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -18,6 +18,7 @@
package org.apache.arrow.vector.complex.impl;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.NonNullableStructVector;
import org.apache.arrow.vector.complex.StateTool;
@@ -25,8 +26,6 @@ import org.apache.arrow.vector.complex.StructVector;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
import org.apache.arrow.vector.types.pojo.Field;
-import com.google.common.base.Preconditions;
-
public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
private NullableStructWriter structRoot;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
index a5535bd..47cb0eb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleStructReaderImpl.java
@@ -20,6 +20,7 @@
package org.apache.arrow.vector.complex.impl;
+import java.util.HashMap;
import java.util.Map;
import org.apache.arrow.vector.ValueVector;
@@ -29,13 +30,11 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.StructWriter;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
-import com.google.common.collect.Maps;
-
@SuppressWarnings("unused")
public class SingleStructReaderImpl extends AbstractFieldReader {
private final NonNullableStructVector vector;
- private final Map<String, FieldReader> fields = Maps.newHashMap();
+ private final Map<String, FieldReader> fields = new HashMap<>();
public SingleStructReaderImpl(NonNullableStructVector vector) {
this.vector = vector;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
index e59f385..44fa608 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/dictionary/DictionaryEncoder.java
@@ -21,6 +21,7 @@ package org.apache.arrow.vector.dictionary;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
@@ -31,8 +32,6 @@ import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.TransferPair;
-import com.google.common.collect.ImmutableList;
-
public class DictionaryEncoder {
// TODO recursively examine fields?
@@ -64,7 +63,7 @@ public class DictionaryEncoder {
// use reflection to pull out the set method
// TODO implement a common interface for int vectors
Method setter = null;
- for (Class<?> c : ImmutableList.of(int.class, long.class)) {
+ for (Class<?> c : Arrays.asList(int.class, long.class)) {
try {
setter = indices.getClass().getMethod("setSafe", int.class, c);
break;
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
index a968eb7..3bfe9c3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowFileWriter.java
@@ -23,6 +23,7 @@ import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
import java.util.List;
+import org.apache.arrow.util.VisibleForTesting;
import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.ArrowBlock;
@@ -32,8 +33,6 @@ import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.annotations.VisibleForTesting;
-
public class ArrowFileWriter extends ArrowWriter {
private static final Logger LOGGER = LoggerFactory.getLogger(ArrowFileWriter.class);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
index 1e8dc82..5f7459e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowReader.java
@@ -37,8 +37,6 @@ import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
import org.apache.arrow.vector.util.DictionaryUtility;
-import com.google.common.collect.ImmutableList;
-
/**
* Abstract class to read Schema and ArrowRecordBatches.
*
@@ -237,7 +235,9 @@ public abstract class ArrowReader implements DictionaryProvider, AutoCloseable {
throw new IllegalArgumentException("Dictionary ID " + id + " not defined in schema");
}
FieldVector vector = dictionary.getVector();
- VectorSchemaRoot root = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector), 0);
+ VectorSchemaRoot root = new VectorSchemaRoot(
+ Collections.singletonList(vector.getField()),
+ Collections.singletonList(vector), 0);
VectorLoader loader = new VectorLoader(root);
try {
loader.load(dictionaryBatch.getDictionary());
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
index 43f1698..894be31 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/ArrowWriter.java
@@ -21,6 +21,7 @@ package org.apache.arrow.vector.ipc;
import java.io.IOException;
import java.nio.channels.WritableByteChannel;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -40,8 +41,6 @@ import org.apache.arrow.vector.util.DictionaryUtility;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableList;
-
/**
* Abstract base class for implementing Arrow writers for IPC over a WriteChannel
*/
@@ -84,7 +83,9 @@ public abstract class ArrowWriter implements AutoCloseable {
Dictionary dictionary = provider.lookup(id);
FieldVector vector = dictionary.getVector();
int count = vector.getValueCount();
- VectorSchemaRoot dictRoot = new VectorSchemaRoot(ImmutableList.of(vector.getField()), ImmutableList.of(vector),
+ VectorSchemaRoot dictRoot = new VectorSchemaRoot(
+ Collections.singletonList(vector.getField()),
+ Collections.singletonList(vector),
count);
VectorUnloader unloader = new VectorUnloader(dictRoot);
ArrowRecordBatch batch = unloader.getRecordBatch();
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
index e4f4e57..b16862c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileReader.java
@@ -23,21 +23,35 @@ import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
import static com.fasterxml.jackson.core.JsonToken.START_ARRAY;
import static com.fasterxml.jackson.core.JsonToken.START_OBJECT;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.arrow.vector.BufferLayout.BufferType.*;
+import static org.apache.arrow.vector.BufferLayout.BufferType.DATA;
+import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET;
+import static org.apache.arrow.vector.BufferLayout.BufferType.TYPE;
+import static org.apache.arrow.vector.BufferLayout.BufferType.VALIDITY;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
-import java.util.*;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.BufferLayout.BufferType;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TinyIntVector;
import org.apache.arrow.vector.TypeLayout;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.Dictionary;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
@@ -53,9 +67,6 @@ import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonParser;
import com.fasterxml.jackson.core.JsonToken;
import com.fasterxml.jackson.databind.MappingJsonFactory;
-import com.google.common.base.Objects;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
import io.netty.buffer.ArrowBuf;
@@ -125,8 +136,8 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
// Read the dictionary record batch
nextFieldIs("data");
FieldVector vector = dict.getVector();
- List<Field> fields = ImmutableList.of(vector.getField());
- List<FieldVector> vectors = ImmutableList.of(vector);
+ List<Field> fields = Collections.singletonList(vector.getField());
+ List<FieldVector> vectors = Collections.singletonList(vector);
VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount());
read(root);
@@ -334,7 +345,7 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
BufferReader FIXEDSIZEBINARY = new BufferReader() {
@Override
protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
- ArrayList<byte[]> values = Lists.newArrayList();
+ ArrayList<byte[]> values = new ArrayList<>();
for (int i = 0; i < count; i++) {
parser.nextToken();
final byte[] value = decodeHexSafe(parser.readValueAs(String.class));
@@ -354,7 +365,7 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
BufferReader VARCHAR = new BufferReader() {
@Override
protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
- ArrayList<byte[]> values = Lists.newArrayList();
+ ArrayList<byte[]> values = new ArrayList<>();
int bufferSize = 0;
for (int i = 0; i < count; i++) {
parser.nextToken();
@@ -377,7 +388,7 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
BufferReader VARBINARY = new BufferReader() {
@Override
protected ArrowBuf read(BufferAllocator allocator, int count) throws IOException {
- ArrayList<byte[]> values = Lists.newArrayList();
+ ArrayList<byte[]> values = new ArrayList<>();
int bufferSize = 0;
for (int i = 0; i < count; i++) {
parser.nextToken();
@@ -519,7 +530,7 @@ public class JsonFileReader implements AutoCloseable, DictionaryProvider {
{
// If currently reading dictionaries, field name is not important so don't check
String name = readNextField("name", String.class);
- if (started && !Objects.equal(field.getName(), name)) {
+ if (started && !Objects.equals(field.getName(), name)) {
throw new IllegalArgumentException("Expected field " + field.getName() + " but got " + name);
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
index d294277..614eb08 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/JsonFileWriter.java
@@ -18,19 +18,48 @@
package org.apache.arrow.vector.ipc;
-import static org.apache.arrow.vector.BufferLayout.BufferType.*;
+import static org.apache.arrow.vector.BufferLayout.BufferType.DATA;
+import static org.apache.arrow.vector.BufferLayout.BufferType.OFFSET;
+import static org.apache.arrow.vector.BufferLayout.BufferType.TYPE;
+import static org.apache.arrow.vector.BufferLayout.BufferType.VALIDITY;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.arrow.vector.*;
+import org.apache.arrow.vector.BaseVariableWidthVector;
+import org.apache.arrow.vector.BigIntVector;
+import org.apache.arrow.vector.BitVectorHelper;
import org.apache.arrow.vector.BufferLayout.BufferType;
+import org.apache.arrow.vector.DateDayVector;
+import org.apache.arrow.vector.DateMilliVector;
+import org.apache.arrow.vector.DecimalVector;
+import org.apache.arrow.vector.FieldVector;
+import org.apache.arrow.vector.FixedSizeBinaryVector;
+import org.apache.arrow.vector.Float4Vector;
+import org.apache.arrow.vector.Float8Vector;
+import org.apache.arrow.vector.IntVector;
+import org.apache.arrow.vector.SmallIntVector;
+import org.apache.arrow.vector.TimeMicroVector;
+import org.apache.arrow.vector.TimeMilliVector;
+import org.apache.arrow.vector.TimeNanoVector;
+import org.apache.arrow.vector.TimeSecVector;
+import org.apache.arrow.vector.TimeStampMicroTZVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampMilliTZVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampNanoTZVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
+import org.apache.arrow.vector.TimeStampSecTZVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TinyIntVector;
import org.apache.arrow.vector.TypeLayout;
+import org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.arrow.vector.dictionary.Dictionary;
import org.apache.arrow.vector.dictionary.DictionaryProvider;
import org.apache.arrow.vector.types.Types.MinorType;
@@ -45,7 +74,6 @@ import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter;
import com.fasterxml.jackson.core.util.DefaultPrettyPrinter.NopIndenter;
import com.fasterxml.jackson.databind.MappingJsonFactory;
-import com.google.common.collect.ImmutableList;
import io.netty.buffer.ArrowBuf;
@@ -123,8 +151,8 @@ public class JsonFileWriter implements AutoCloseable {
generator.writeFieldName("data");
Dictionary dictionary = provider.lookup(id);
FieldVector vector = dictionary.getVector();
- List<Field> fields = ImmutableList.of(vector.getField());
- List<FieldVector> vectors = ImmutableList.of(vector);
+ List<Field> fields = Collections.singletonList(vector.getField());
+ List<FieldVector> vectors = Collections.singletonList(vector);
VectorSchemaRoot root = new VectorSchemaRoot(fields, vectors, vector.getValueCount());
writeBatch(root);
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
index 83d7c06..1e56e97 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ipc/message/ArrowRecordBatch.java
@@ -22,8 +22,10 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.stream.Collectors;
import org.apache.arrow.flatbuf.RecordBatch;
+import org.apache.arrow.memory.BufferAllocator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,6 +82,22 @@ public class ArrowRecordBatch implements ArrowMessage {
this.buffersLayout = Collections.unmodifiableList(arrowBuffers);
}
+ // clone constructor
+ private ArrowRecordBatch(boolean dummy, int length, List<ArrowFieldNode> nodes, List<ArrowBuf> buffers) {
+ this.length = length;
+ this.nodes = nodes;
+ this.buffers = buffers;
+ this.closed = false;
+ List<ArrowBuffer> arrowBuffers = new ArrayList<>();
+ long offset = 0;
+ for (ArrowBuf arrowBuf : buffers) {
+ long size = arrowBuf.readableBytes();
+ arrowBuffers.add(new ArrowBuffer(offset, size));
+ offset += size;
+ }
+ this.buffersLayout = Collections.unmodifiableList(arrowBuffers);
+ }
+
public int getLength() {
return length;
}
@@ -102,6 +120,22 @@ public class ArrowRecordBatch implements ArrowMessage {
}
/**
+ * Create a new ArrowRecordBatch which has the same information as this batch but whose buffers
+ * are owned by that Allocator.
+ *
+ * This will also close this record batch and make it no longer useful.
+ *
+ * @return A cloned ArrowRecordBatch
+ */
+ public ArrowRecordBatch cloneWithTransfer(final BufferAllocator allocator) {
+ final List<ArrowBuf> newBufs = buffers.stream()
+ .map(t -> (t.transferOwnership(allocator).buffer).writerIndex(t.writerIndex()))
+ .collect(Collectors.toList());
+ close();
+ return new ArrowRecordBatch(false, length, nodes, newBufs);
+ }
+
+ /**
* @return the serialized layout if we send the buffers on the wire
*/
public List<ArrowBuffer> getBuffersLayout() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
index 78f20d4..1f5f76d 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Field.java
@@ -18,15 +18,19 @@
package org.apache.arrow.vector.types.pojo;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.arrow.util.Preconditions.checkNotNull;
import static org.apache.arrow.vector.complex.BaseRepeatedValueVector.DATA_VECTOR_NAME;
import static org.apache.arrow.vector.types.pojo.ArrowType.getTypeForField;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.stream.Collectors;
import org.apache.arrow.flatbuf.KeyValue;
import org.apache.arrow.flatbuf.Type;
@@ -40,9 +44,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.flatbuffers.FlatBufferBuilder;
public class Field {
@@ -73,7 +74,7 @@ public class Field {
private Field(String name, FieldType fieldType, List<Field> children, TypeLayout typeLayout) {
this.name = name;
this.fieldType = checkNotNull(fieldType);
- this.children = children == null ? ImmutableList.<Field>of() : ImmutableList.copyOf(children);
+ this.children = children == null ? Collections.emptyList() : children.stream().collect(Collectors.toList());
}
// deprecated, use FieldType or static constructor instead
@@ -112,20 +113,20 @@ public class Field {
}
dictionary = new DictionaryEncoding(dictionaryFB.id(), dictionaryFB.isOrdered(), indexType);
}
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ List<Field> children = new ArrayList<>();
for (int i = 0; i < field.childrenLength(); i++) {
Field childField = convertField(field.children(i));
childField = mutateOriginalNameIfNeeded(field, childField);
- childrenBuilder.add(childField);
+ children.add(childField);
}
- List<Field> children = childrenBuilder.build();
- ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
+ children = Collections.unmodifiableList(children);
+ Map<String, String> metadata = new HashMap<>();
for (int i = 0; i < field.customMetadataLength(); i++) {
KeyValue kv = field.customMetadata(i);
String key = kv.key(), value = kv.value();
- metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
+ metadata.put(key == null ? "" : key, value == null ? "" : value);
}
- Map<String, String> metadata = metadataBuilder.build();
+ metadata = Collections.unmodifiableMap(metadata);
return new Field(name, nullable, type, dictionary, children, metadata);
}
@@ -256,7 +257,10 @@ public class Field {
sb.append("[dictionary: ").append(getDictionary().getId()).append("]");
}
if (!children.isEmpty()) {
- sb.append("<").append(Joiner.on(", ").join(children)).append(">");
+ sb.append("<").append(children.stream()
+ .map(t -> t.toString())
+ .collect(Collectors.joining(", ")))
+ .append(">");
}
if (!isNullable()) {
sb.append(" not null");
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
index 6422f39..5030162 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/FieldType.java
@@ -18,18 +18,16 @@
package org.apache.arrow.vector.types.pojo;
-import static com.google.common.base.Preconditions.checkNotNull;
-
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Collections2;
+import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.CallBack;
-import com.google.common.collect.ImmutableMap;
-
public class FieldType {
public static FieldType nullable(ArrowType type) {
@@ -48,9 +46,9 @@ public class FieldType {
public FieldType(boolean nullable, ArrowType type, DictionaryEncoding dictionary, Map<String, String> metadata) {
super();
this.nullable = nullable;
- this.type = checkNotNull(type);
+ this.type = Preconditions.checkNotNull(type);
this.dictionary = dictionary;
- this.metadata = metadata == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(metadata);
+ this.metadata = metadata == null ? java.util.Collections.emptyMap() : Collections2.immutableMapCopy(metadata);
}
public boolean isNullable() {
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
index fca93ae..6dcfaea 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/pojo/Schema.java
@@ -19,20 +19,22 @@
package org.apache.arrow.vector.types.pojo;
-import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.arrow.vector.types.pojo.Field.convertField;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
-import java.util.Collections;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
+import java.util.stream.Collectors;
import org.apache.arrow.flatbuf.KeyValue;
+import org.apache.arrow.util.Collections2;
+import org.apache.arrow.util.Preconditions;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonInclude;
@@ -42,9 +44,6 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectReader;
import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Joiner;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import com.google.flatbuffers.FlatBufferBuilder;
/**
@@ -72,7 +71,7 @@ public class Schema {
private static final ObjectReader reader = mapper.readerFor(Schema.class);
public static Schema fromJSON(String json) throws IOException {
- return reader.readValue(checkNotNull(json));
+ return reader.readValue(Preconditions.checkNotNull(json));
}
public static Schema deserialize(ByteBuffer buffer) {
@@ -80,19 +79,17 @@ public class Schema {
}
public static Schema convertSchema(org.apache.arrow.flatbuf.Schema schema) {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ List<Field> fields = new ArrayList<>();
for (int i = 0; i < schema.fieldsLength(); i++) {
- childrenBuilder.add(convertField(schema.fields(i)));
+ fields.add(convertField(schema.fields(i)));
}
- List<Field> fields = childrenBuilder.build();
- ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
+ Map<String, String> metadata = new HashMap<>();
for (int i = 0; i < schema.customMetadataLength(); i++) {
KeyValue kv = schema.customMetadata(i);
String key = kv.key(), value = kv.value();
- metadataBuilder.put(key == null ? "" : key, value == null ? "" : value);
+ metadata.put(key == null ? "" : key, value == null ? "" : value);
}
- Map<String, String> metadata = metadataBuilder.build();
- return new Schema(fields, metadata);
+ return new Schema(Collections2.immutableListCopy(fields), Collections2.immutableMapCopy(metadata));
}
private final List<Field> fields;
@@ -109,8 +106,8 @@ public class Schema {
for (Field field : fields) {
fieldList.add(field);
}
- this.fields = Collections.unmodifiableList(fieldList);
- this.metadata = metadata == null ? ImmutableMap.<String, String>of() : ImmutableMap.copyOf(metadata);
+ this.fields = Collections2.immutableListCopy(fieldList);
+ this.metadata = metadata == null ? java.util.Collections.emptyMap() : Collections2.immutableMapCopy(metadata);
}
public List<Field> getFields() {
@@ -163,6 +160,15 @@ public class Schema {
return org.apache.arrow.flatbuf.Schema.endSchema(builder);
}
+ public byte[] toByteArray() {
+ FlatBufferBuilder builder = new FlatBufferBuilder();
+ int schemaOffset = this.getSchema(builder);
+ builder.finish(schemaOffset);
+ ByteBuffer bb = builder.dataBuffer();
+ byte[] bytes = new byte[bb.remaining()];
+ bb.get(bytes);
+ return bytes;
+ }
@Override
public int hashCode() {
@@ -181,6 +187,6 @@ public class Schema {
@Override
public String toString() {
String meta = metadata.isEmpty() ? "" : "(metadata: " + metadata.toString() + ")";
- return "Schema<" + Joiner.on(", ").join(fields) + ">" + meta;
+ return "Schema<" + fields.stream().map(t -> t.toString()).collect(Collectors.joining(", ")) + ">" + meta;
}
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
index 67ce99b..37fb05e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/ByteFunctionHelpers.java
@@ -21,8 +21,6 @@ package org.apache.arrow.vector.util;
import org.apache.arrow.memory.BoundsChecking;
-import com.google.common.primitives.UnsignedLongs;
-
import io.netty.buffer.ArrowBuf;
import io.netty.util.internal.PlatformDependent;
@@ -125,7 +123,7 @@ public class ByteFunctionHelpers {
long leftLong = PlatformDependent.getLong(lPos);
long rightLong = PlatformDependent.getLong(rPos);
if (leftLong != rightLong) {
- return UnsignedLongs.compare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
+ return unsignedLongCompare(Long.reverseBytes(leftLong), Long.reverseBytes(rightLong));
}
lPos += 8;
rPos += 8;
@@ -175,6 +173,20 @@ public class ByteFunctionHelpers {
}
+ /**
+ * Compares the two specified {@code long} values, treating them as unsigned values between
+ * {@code 0} and {@code 2^64 - 1} inclusive.
+ *
+ * @param a the first unsigned {@code long} to compare
+ * @param b the second unsigned {@code long} to compare
+ * @return a negative value if {@code a} is less than {@code b}; a positive value if {@code a} is
+ * greater than {@code b}; or zero if they are equal
+ */
+ public static int unsignedLongCompare(long a, long b) {
+ return Long.compare(a ^ Long.MIN_VALUE, b ^ Long.MIN_VALUE);
+ }
+
+
private static final int memcmp(
final long laddr,
int lStart,
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
index 8e35979..ad9d693 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/MapWithOrdinal.java
@@ -21,20 +21,17 @@ package org.apache.arrow.vector.util;
import java.util.AbstractMap;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+import org.apache.arrow.util.Preconditions;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
import io.netty.util.collection.IntObjectHashMap;
import io.netty.util.collection.IntObjectMap;
@@ -59,7 +56,7 @@ import io.netty.util.collection.IntObjectMap;
public class MapWithOrdinal<K, V> implements Map<K, V> {
private final static Logger logger = LoggerFactory.getLogger(MapWithOrdinal.class);
- private final Map<K, Entry<Integer, V>> primary = Maps.newLinkedHashMap();
+ private final Map<K, Entry<Integer, V>> primary = new HashMap<>();
private final IntObjectHashMap<V> secondary = new IntObjectHashMap<>();
private final Map<K, V> delegate = new Map<K, V>() {
@@ -134,24 +131,16 @@ public class MapWithOrdinal<K, V> implements Map<K, V> {
@Override
public Collection<V> values() {
- return Lists.newArrayList(Iterables.transform(secondary.entries(),
- new Function<IntObjectMap.PrimitiveEntry<V>, V>() {
- @Override
- public V apply(IntObjectMap.PrimitiveEntry<V> entry) {
- return Preconditions.checkNotNull(entry).value();
- }
- }));
+ return StreamSupport.stream(secondary.entries().spliterator(), false)
+ .map((IntObjectMap.PrimitiveEntry<V> t) -> Preconditions.checkNotNull(t).value())
+ .collect(Collectors.toList());
}
@Override
public Set<Entry<K, V>> entrySet() {
- return Sets.newHashSet(Iterables.transform(primary.entrySet(),
- new Function<Entry<K, Entry<Integer, V>>, Entry<K, V>>() {
- @Override
- public Entry<K, V> apply(Entry<K, Entry<Integer, V>> entry) {
- return new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), entry.getValue().getValue());
- }
- }));
+ return primary.entrySet().stream()
+ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), entry.getValue().getValue()))
+ .collect(Collectors.toSet());
}
};
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
index 624ea1b..2462678 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/util/Validator.java
@@ -20,6 +20,7 @@ package org.apache.arrow.vector.util;
import java.util.Arrays;
import java.util.List;
+import java.util.Objects;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.VectorSchemaRoot;
@@ -30,8 +31,6 @@ import org.apache.arrow.vector.types.pojo.DictionaryEncoding;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.Schema;
-import com.google.common.base.Objects;
-
/**
* Utility class for validating arrow data structures
*/
@@ -154,7 +153,7 @@ public class Validator {
return Arrays.equals((byte[]) o1, (byte[]) o2);
}
- return Objects.equal(o1, o2);
+ return Objects.equals(o1, o2);
}
static boolean equalEnough(Float f1, Float f2) {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
index 0d6f975..f4d8279 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestFixedSizeListVector.java
@@ -18,6 +18,8 @@
package org.apache.arrow.vector;
+import java.util.Arrays;
+
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.ListVector;
@@ -33,8 +35,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.google.common.collect.Lists;
-
public class TestFixedSizeListVector {
private BufferAllocator allocator;
@@ -71,7 +71,7 @@ public class TestFixedSizeListVector {
Assert.assertTrue(reader.next());
Assert.assertEquals(i + 10, reader.reader().readInteger().intValue());
Assert.assertFalse(reader.next());
- Assert.assertEquals(Lists.newArrayList(i, i + 10), reader.readObject());
+ Assert.assertEquals(Arrays.asList(i, i + 10), reader.readObject());
}
}
}
@@ -102,7 +102,7 @@ public class TestFixedSizeListVector {
Assert.assertTrue(reader.next());
Assert.assertEquals(i + 10.1f, reader.reader().readFloat(), 0.00001);
Assert.assertFalse(reader.next());
- Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), reader.readObject());
+ Assert.assertEquals(Arrays.asList(i + 0.1f, i + 10.1f), reader.readObject());
} else {
Assert.assertFalse(reader.isSet());
Assert.assertNull(reader.readObject());
@@ -193,7 +193,7 @@ public class TestFixedSizeListVector {
Assert.assertTrue(reader.next());
Assert.assertEquals(10.1f, reader.reader().readFloat(), 0.00001);
Assert.assertFalse(reader.next());
- Assert.assertEquals(Lists.newArrayList(0.1f, 10.1f), reader.readObject());
+ Assert.assertEquals(Arrays.asList(0.1f, 10.1f), reader.readObject());
reader.setPosition(2);
Assert.assertTrue(reader.isSet());
@@ -202,7 +202,7 @@ public class TestFixedSizeListVector {
Assert.assertTrue(reader.next());
Assert.assertEquals(12.1f, reader.reader().readFloat(), 0.00001);
Assert.assertFalse(reader.next());
- Assert.assertEquals(Lists.newArrayList(2.1f, 12.1f), reader.readObject());
+ Assert.assertEquals(Arrays.asList(2.1f, 12.1f), reader.readObject());
reader.setPosition(3);
Assert.assertTrue(reader.isSet());
@@ -211,7 +211,7 @@ public class TestFixedSizeListVector {
Assert.assertTrue(reader.next());
Assert.assertEquals(14.1f, reader.reader().readFloat(), 0.00001);
Assert.assertFalse(reader.next());
- Assert.assertEquals(Lists.newArrayList(4.1f, 14.1f), reader.readObject());
+ Assert.assertEquals(Arrays.asList(4.1f, 14.1f), reader.readObject());
for (int i = 4; i < 10; i++) {
reader.setPosition(i);
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
index 80da8e7..02b07b7 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/BaseFileTest.java
@@ -31,6 +31,7 @@ import java.util.List;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.Collections2;
import org.apache.arrow.vector.DateMilliVector;
import org.apache.arrow.vector.DecimalVector;
import org.apache.arrow.vector.FieldVector;
@@ -72,8 +73,6 @@ import org.junit.Before;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableList;
-
import io.netty.buffer.ArrowBuf;
/**
@@ -295,9 +294,9 @@ public class BaseFileTest {
FieldVector encodedVector2 = (FieldVector) DictionaryEncoder.encode(vector2, dictionary2);
vector2.close(); // Done with this vector after encoding
- List<Field> fields = ImmutableList.of(encodedVector1A.getField(), encodedVector1B.getField(),
+ List<Field> fields = Arrays.asList(encodedVector1A.getField(), encodedVector1B.getField(),
encodedVector2.getField());
- List<FieldVector> vectors = ImmutableList.of(encodedVector1A, encodedVector1B, encodedVector2);
+ List<FieldVector> vectors = Collections2.asImmutableList(encodedVector1A, encodedVector1B, encodedVector2);
return new VectorSchemaRoot(fields, vectors, encodedVector1A.getValueCount());
}
@@ -398,8 +397,8 @@ public class BaseFileTest {
listWriter.endList();
listWriter.setValueCount(3);
- List<Field> fields = ImmutableList.of(listVector.getField());
- List<FieldVector> vectors = ImmutableList.<FieldVector>of(listVector);
+ List<Field> fields = Collections2.asImmutableList(listVector.getField());
+ List<FieldVector> vectors = Collections2.asImmutableList(listVector);
return new VectorSchemaRoot(fields, vectors, 3);
}
@@ -447,9 +446,9 @@ public class BaseFileTest {
decimalVector2.setValueCount(count);
decimalVector3.setValueCount(count);
- List<Field> fields = ImmutableList.of(decimalVector1.getField(), decimalVector2.getField(),
+ List<Field> fields = Collections2.asImmutableList(decimalVector1.getField(), decimalVector2.getField(),
decimalVector3.getField());
- List<FieldVector> vectors = ImmutableList.<FieldVector>of(decimalVector1, decimalVector2, decimalVector3);
+ List<FieldVector> vectors = Collections2.asImmutableList(decimalVector1, decimalVector2, decimalVector3);
return new VectorSchemaRoot(fields, vectors, count);
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
index 950976c..f89f378 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/ipc/TestArrowFile.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.util.Collections2;
import org.apache.arrow.vector.FieldVector;
import org.apache.arrow.vector.FixedSizeBinaryVector;
import org.apache.arrow.vector.Float4Vector;
@@ -59,10 +60,6 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
public class TestArrowFile extends BaseFileTest {
private static final Logger LOGGER = LoggerFactory.getLogger(TestArrowFile.class);
@@ -373,18 +370,18 @@ public class TestArrowFile extends BaseFileTest {
File file = new File("target/mytest_metadata.arrow");
ByteArrayOutputStream stream = new ByteArrayOutputStream();
- List<Field> childFields = new ArrayList<Field>();
+ List<Field> childFields = new ArrayList<>();
childFields.add(new Field("varchar-child", new FieldType(true, ArrowType.Utf8.INSTANCE, null, metadata(1)), null));
childFields.add(new Field("float-child",
new FieldType(true, new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE), null, metadata(2)), null));
childFields.add(new Field("int-child", new FieldType(false, new ArrowType.Int(32, true), null, metadata(3)), null));
childFields.add(new Field("list-child", new FieldType(true, ArrowType.List.INSTANCE, null, metadata(4)),
- ImmutableList.of(new Field("l1", FieldType.nullable(new ArrowType.Int(16, true)), null))));
+ Collections2.asImmutableList(new Field("l1", FieldType.nullable(new ArrowType.Int(16, true)), null))));
Field field = new Field("meta", new FieldType(true, ArrowType.Struct.INSTANCE, null, metadata(0)), childFields);
Map<String, String> metadata = new HashMap<>();
metadata.put("s1", "v1");
metadata.put("s2", "v2");
- Schema originalSchema = new Schema(ImmutableList.of(field), metadata);
+ Schema originalSchema = new Schema(Collections2.asImmutableList(field), metadata);
Assert.assertEquals(metadata, originalSchema.getCustomMetadata());
// write
@@ -394,7 +391,7 @@ public class TestArrowFile extends BaseFileTest {
vector.allocateNewSafe();
vector.setValueCount(0);
- List<FieldVector> vectors = ImmutableList.<FieldVector>of(vector);
+ List<FieldVector> vectors = Collections2.asImmutableList(vector);
VectorSchemaRoot root = new VectorSchemaRoot(originalSchema, vectors, 0);
try (FileOutputStream fileOutputStream = new FileOutputStream(file);
@@ -444,7 +441,10 @@ public class TestArrowFile extends BaseFileTest {
}
private Map<String, String> metadata(int i) {
- return ImmutableMap.of("k_" + i, "v_" + i, "k2_" + i, "v2_" + i);
+ Map<String, String> map = new HashMap<>();
+ map.put("k_" + i, "v_" + i);
+ map.put("k2_" + i, "v2_" + i);
+ return Collections.unmodifiableMap(map);
}
@Test
@@ -661,7 +661,8 @@ public class TestArrowFile extends BaseFileTest {
arrowReader.loadRecordBatch(rbBlock);
Assert.assertEquals(count, root.getRowCount());
for (int i = 0; i < 10; i++) {
- Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i));
+ Assert.assertEquals(Collections2.asImmutableList(i + 0.1f, i + 10.1f), root.getVector("float-pairs")
+ .getObject(i));
Assert.assertEquals(i, root.getVector("ints").getObject(i));
}
}
@@ -677,7 +678,8 @@ public class TestArrowFile extends BaseFileTest {
arrowReader.loadNextBatch();
Assert.assertEquals(count, root.getRowCount());
for (int i = 0; i < 10; i++) {
- Assert.assertEquals(Lists.newArrayList(i + 0.1f, i + 10.1f), root.getVector("float-pairs").getObject(i));
+ Assert.assertEquals(Collections2.asImmutableList(i + 0.1f, i + 10.1f), root.getVector("float-pairs")
+ .getObject(i));
Assert.assertEquals(i, root.getVector("ints").getObject(i));
}
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 2c4cf36..4bb614e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -24,11 +24,14 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.util.Collections2;
import org.apache.arrow.vector.complex.FixedSizeListVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.types.TimeUnit;
@@ -46,7 +49,6 @@ import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.types.pojo.Schema;
import org.junit.Test;
-import com.google.common.collect.ImmutableList;
import com.google.flatbuffers.FlatBufferBuilder;
/**
@@ -62,29 +64,29 @@ public class TestConvert {
@Test
public void complex() {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
- childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
+ java.util.List<Field> children = new ArrayList<>();
+ children.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ children.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), Collections.emptyList()));
- Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build());
+ Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), children);
run(initialField);
}
@Test
public void list() throws Exception {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
+ java.util.List<Field> children = new ArrayList<>();
try (BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
ListVector writeVector = ListVector.empty("list", allocator);
FixedSizeListVector writeFixedVector = FixedSizeListVector.empty("fixedlist", 5, allocator)) {
Field listVectorField = writeVector.getField();
- childrenBuilder.add(listVectorField);
+ children.add(listVectorField);
Field listFixedVectorField = writeFixedVector.getField();
- childrenBuilder.add(listFixedVectorField);
+ children.add(listFixedVectorField);
}
- Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), childrenBuilder.build());
- ImmutableList.Builder<Field> parentBuilder = ImmutableList.builder();
- parentBuilder.add(initialField);
+ Field initialField = new Field("a", FieldType.nullable(Struct.INSTANCE), children);
+ java.util.List<Field> parent = new ArrayList<>();
+ parent.add(initialField);
FlatBufferBuilder builder = new FlatBufferBuilder();
builder.finish(initialField.getField(builder));
org.apache.arrow.flatbuf.Field flatBufField = org.apache.arrow.flatbuf.Field.getRootAsField(builder.dataBuffer());
@@ -92,7 +94,7 @@ public class TestConvert {
assertEquals(initialField, finalField);
assertFalse(finalField.toString().contains("[DEFAULT]"));
- Schema initialSchema = new Schema(parentBuilder.build());
+ Schema initialSchema = new Schema(parent);
String jsonSchema = initialSchema.toJson();
String modifiedSchema = jsonSchema.replace("$data$", "[DEFAULT]");
@@ -107,45 +109,45 @@ public class TestConvert {
@Test
public void schema() {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
- childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
- Schema initialSchema = new Schema(childrenBuilder.build());
+ java.util.List<Field> children = new ArrayList<>();
+ children.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ children.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), Collections.emptyList()));
+ Schema initialSchema = new Schema(children);
run(initialSchema);
}
@Test
public void schemaMetadata() {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
- childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
+ java.util.List<Field> children = new ArrayList<>();
+ children.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ children.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), Collections.emptyList()));
Map<String, String> metadata = new HashMap<>();
metadata.put("key1", "value1");
metadata.put("key2", "value2");
- Schema initialSchema = new Schema(childrenBuilder.build(), metadata);
+ Schema initialSchema = new Schema(children, metadata);
run(initialSchema);
}
@Test
public void nestedSchema() {
- ImmutableList.Builder<Field> childrenBuilder = ImmutableList.builder();
- childrenBuilder.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
- childrenBuilder.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), ImmutableList.<Field>of()));
- childrenBuilder.add(new Field("child3", FieldType.nullable(new Struct()), ImmutableList.<Field>of(
+ java.util.List<Field> children = new ArrayList<>();
+ children.add(new Field("child1", FieldType.nullable(Utf8.INSTANCE), null));
+ children.add(new Field("child2", FieldType.nullable(new FloatingPoint(SINGLE)), Collections.emptyList()));
+ children.add(new Field("child3", FieldType.nullable(new Struct()), Collections2.asImmutableList(
new Field("child3.1", FieldType.nullable(Utf8.INSTANCE), null),
- new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of())
+ new Field("child3.2", FieldType.nullable(new FloatingPoint(DOUBLE)), Collections.emptyList())
)));
- childrenBuilder.add(new Field("child4", FieldType.nullable(new List()), ImmutableList.<Field>of(
+ children.add(new Field("child4", FieldType.nullable(new List()), Collections2.asImmutableList(
new Field("child4.1", FieldType.nullable(Utf8.INSTANCE), null)
)));
- childrenBuilder.add(new Field("child5", FieldType.nullable(
+ children.add(new Field("child5", FieldType.nullable(
new Union(UnionMode.Sparse, new int[] {MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal()})),
- ImmutableList.<Field>of(
+ Collections2.asImmutableList(
new Field("child5.1", FieldType.nullable(new Timestamp(TimeUnit.MILLISECOND, null)), null),
- new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), ImmutableList.<Field>of()),
+ new Field("child5.2", FieldType.nullable(new FloatingPoint(DOUBLE)), Collections.emptyList()),
new Field("child5.3", true, new Timestamp(TimeUnit.MILLISECOND, "UTC"), null)
)));
- Schema initialSchema = new Schema(childrenBuilder.build());
+ Schema initialSchema = new Schema(children);
run(initialSchema);
}