You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2019/11/12 10:19:24 UTC
[tinkerpop] branch TINKERPOP-2076 updated (d4c8669 -> a3fd3d4)
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a change to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git.
from d4c8669 TINKERPOP-2076 Fixed Gremlin Server integration tests after Java 11
new 091e44d Reverts TINKERPOP-2289 resolve ip address
new c4189ac TINKERPOP-2076 Fixed mockito usage after rebase
new 6833c7b TINKERPOP-2076 Bump to spark 3.0 with jdk11 support
new da80e23 TINKERPOP-2076 Removed a scala class registration from gryo
new a3fd3d4 TINKERPOP-2076 Fixed bad merge operation in the MemoryAccumulator
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGELOG.asciidoc | 4 +-
gremlin-driver/pom.xml | 18 --
.../apache/tinkerpop/gremlin/driver/Cluster.java | 24 +-
.../org/apache/tinkerpop/gremlin/driver/Host.java | 3 +-
.../apache/tinkerpop/gremlin/driver/HostTest.java | 46 +---
.../binary/GraphBinaryMessageSerializerV1Test.java | 7 +-
.../GraphBinaryReaderWriterRoundTripTest.java | 11 +-
.../types/sample/SamplePersonSerializerTest.java | 5 +-
gremlin-groovy/pom.xml | 2 +-
pom.xml | 2 +-
spark-gremlin/pom.xml | 254 ++++-----------------
.../spark/process/computer/MemoryAccumulator.java | 47 ++--
.../spark/process/computer/SparkMemory.java | 16 +-
.../spark/structure/io/gryo/GryoRegistrator.java | 1 -
.../spark/structure/io/gryo/GryoSerializer.java | 1 -
15 files changed, 117 insertions(+), 324 deletions(-)
[tinkerpop] 04/05: TINKERPOP-2076 Removed a scala class
registration from gryo
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit da80e23cf67f06bded4c2a8bc3493423788fc601
Author: stephen <sp...@gmail.com>
AuthorDate: Tue Nov 12 05:04:30 2019 -0500
TINKERPOP-2076 Removed a scala class registration from gryo
Not sure why that class was necessary as test seem to pass without it. I guess if it comes up again later, we'll just add it back.
---
.../tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java | 1 -
.../apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java | 1 -
2 files changed, 2 deletions(-)
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
index 13fc787..d1f7725 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoRegistrator.java
@@ -199,7 +199,6 @@ public class GryoRegistrator implements KryoRegistrator {
// of the entries are for TinkerPop (and the ordering is significant).
try {
m.put(Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
- m.put(Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
m.put(Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), new JavaSerializer());
m.put(Class.forName("org.apache.spark.internal.io.FileCommitProtocol$EmptyTaskCommitMessage$"), new JavaSerializer());
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
index 563c158..6fba103 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/GryoSerializer.java
@@ -139,7 +139,6 @@ public final class GryoSerializer extends Serializer implements Serializable {
super.register(GryoIo.class, PythonBroadcast.class, null);
super.register(GryoIo.class, BoxedUnit.class, null);
super.register(GryoIo.class, Class.forName("scala.reflect.ClassTag$$anon$1"), new JavaSerializer());
- super.register(GryoIo.class, Class.forName("scala.reflect.ManifestFactory$$anon$1"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("org.apache.spark.internal.io.FileCommitProtocol$TaskCommitMessage"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("org.apache.spark.internal.io.FileCommitProtocol$EmptyTaskCommitMessage$"), new JavaSerializer());
super.register(GryoIo.class, Class.forName("scala.collection.immutable.Map$EmptyMap$"), new JavaSerializer());
[tinkerpop] 02/05: TINKERPOP-2076 Fixed mockito usage after rebase
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit c4189ac4425129b65fdd62894fdb41fa3fcfc74d
Author: stephen <sp...@gmail.com>
AuthorDate: Mon Nov 11 21:06:30 2019 -0500
TINKERPOP-2076 Fixed mockito usage after rebase
---
.../driver/ser/binary/GraphBinaryMessageSerializerV1Test.java | 7 ++++---
.../ser/binary/GraphBinaryReaderWriterRoundTripTest.java | 11 ++++++-----
.../ser/binary/types/sample/SamplePersonSerializerTest.java | 5 +++--
3 files changed, 13 insertions(+), 10 deletions(-)
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryMessageSerializerV1Test.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryMessageSerializerV1Test.java
index d0042ff..45a6a13 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryMessageSerializerV1Test.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryMessageSerializerV1Test.java
@@ -35,6 +35,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -54,7 +55,7 @@ public class GraphBinaryMessageSerializerV1Test {
final int mimeLen = buffer.readByte();
buffer.readBytes(new byte[mimeLen]);
final RequestMessage deserialized = serializer.deserializeRequest(buffer);
- assertThat(request, new ReflectionEquals(deserialized));
+ assertThat(new ReflectionEquals(deserialized).matches(request), is(true));
}
@Test
@@ -68,7 +69,7 @@ public class GraphBinaryMessageSerializerV1Test {
final int mimeLen = buffer.readByte();
buffer.readBytes(new byte[mimeLen]);
final RequestMessage deserialized = serializer.deserializeRequest(buffer);
- assertThat(request, new ReflectionEquals(deserialized));
+ assertThat(new ReflectionEquals(deserialized).matches(request), is(true));
}
@Test
@@ -82,7 +83,7 @@ public class GraphBinaryMessageSerializerV1Test {
final int mimeLen = buffer.readByte();
buffer.readBytes(new byte[mimeLen]);
final RequestMessage deserialized = serializer.deserializeRequest(buffer);
- assertThat(request, new ReflectionEquals(deserialized));
+ assertThat(new ReflectionEquals(deserialized).matches(request), is(true));
}
@Test
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryReaderWriterRoundTripTest.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryReaderWriterRoundTripTest.java
index e74e6dc..0a26367 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryReaderWriterRoundTripTest.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/GraphBinaryReaderWriterRoundTripTest.java
@@ -87,6 +87,7 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.hasLabel;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertThat;
@@ -265,19 +266,19 @@ public class GraphBinaryReaderWriterRoundTripTest {
new Object[] {"BulkSet", bulkSet, null},
new Object[] {"Tree", tree, null},
new Object[] {"EmptyMetrics", new MutableMetrics("idEmpty", "nameEmpty"), (Consumer<Metrics>) m -> {
- assertThat(m, new ReflectionEquals(new MutableMetrics("idEmpty", "nameEmpty")));
+ assertThat(new ReflectionEquals(new MutableMetrics("idEmpty", "nameEmpty")).matches(m), is(true));
}},
new Object[] {"Metrics", metrics, (Consumer<Metrics>) m -> {
- assertThat(m, new ReflectionEquals(metrics, "nested", "counts"));
+ assertThat(new ReflectionEquals(metrics, "nested", "counts").matches(m), is(true));
assertEquals(new ArrayList(metrics.getCounts().values()), new ArrayList(m.getCounts().values()));
- assertThat(m.getNested(), new ReflectionEquals(metrics.getNested()));
+ assertThat(new ReflectionEquals(metrics.getNested()).matches(m.getNested()), is(true));
}},
new Object[] {"EmptyTraversalMetrics", emptyTraversalMetrics, (Consumer<TraversalMetrics>) m -> {
- assertThat(m, new ReflectionEquals(emptyTraversalMetrics));
+ assertThat(new ReflectionEquals(emptyTraversalMetrics).matches(m), is(true));
}},
new Object[] {"TraversalMetrics", traversalMetrics, (Consumer<TraversalMetrics>) m -> {
assertEquals(m.toString(), traversalMetrics.toString());
- assertThat(m, new ReflectionEquals(traversalMetrics, "stepIndexedMetrics", "positionIndexedMetrics"));
+ assertThat(new ReflectionEquals(traversalMetrics, "stepIndexedMetrics", "positionIndexedMetrics").matches(m), is(true));
}},
// collections
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/types/sample/SamplePersonSerializerTest.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/types/sample/SamplePersonSerializerTest.java
index 3d6bef3..1db2bf7 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/types/sample/SamplePersonSerializerTest.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/ser/binary/types/sample/SamplePersonSerializerTest.java
@@ -43,6 +43,7 @@ import java.util.UUID;
import static org.apache.tinkerpop.gremlin.driver.ser.AbstractMessageSerializer.TOKEN_IO_REGISTRIES;
import static org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1.TOKEN_CUSTOM;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.assertThat;
public class SamplePersonSerializerTest {
@@ -93,7 +94,7 @@ public class SamplePersonSerializerTest {
writer.writeValue(person, buffer, nullable);
final SamplePerson actual = reader.readValue(buffer, SamplePerson.class, nullable);
- assertThat(actual, new ReflectionEquals(person));
+ assertThat(new ReflectionEquals(person).matches(actual), is(true));
buffer.release();
}
}
@@ -108,7 +109,7 @@ public class SamplePersonSerializerTest {
final ResponseMessage deserialized = serializer.deserializeResponse(serialized);
final SamplePerson actual = (SamplePerson) deserialized.getResult().getData();
- assertThat(actual, new ReflectionEquals(person));
+ assertThat(new ReflectionEquals(person).matches(actual), is(true));
}
public static class CustomIoRegistry extends AbstractIoRegistry {
[tinkerpop] 01/05: Reverts TINKERPOP-2289 resolve ip address
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 091e44d8761c03f506902d1a551c688a3057ea07
Author: Robert Dale <ro...@gmail.com>
AuthorDate: Tue Nov 5 22:39:40 2019 -0500
Reverts TINKERPOP-2289 resolve ip address
---
CHANGELOG.asciidoc | 1 +
gremlin-driver/pom.xml | 18 ---------
.../apache/tinkerpop/gremlin/driver/Cluster.java | 24 ++++-------
.../org/apache/tinkerpop/gremlin/driver/Host.java | 3 +-
.../apache/tinkerpop/gremlin/driver/HostTest.java | 46 +---------------------
5 files changed, 11 insertions(+), 81 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 2bfd4d1..45d4431 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -523,6 +523,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
* Bump to Tornado 5.x for gremlin-python.
* Deprecated `TraversalStrategies.applyStrategies()`.
+* Reverted: Modified Java driver to use IP address rather than hostname to create connections.
[[release-3-3-9]]
=== TinkerPop 3.3.9 (Release Date: October 14, 2019)
diff --git a/gremlin-driver/pom.xml b/gremlin-driver/pom.xml
index 9e3af43..0473e61 100644
--- a/gremlin-driver/pom.xml
+++ b/gremlin-driver/pom.xml
@@ -98,24 +98,6 @@ limitations under the License.
<artifactId>hamcrest-all</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-api-mockito</artifactId>
- <version>1.6.2</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4</artifactId>
- <version>1.6.2</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.powermock</groupId>
- <artifactId>powermock-module-junit4-rule</artifactId>
- <version>1.6.2</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java
index 893ce7b..9bc3eb7 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Cluster.java
@@ -54,7 +54,6 @@ import java.security.NoSuchAlgorithmException;
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
@@ -901,21 +900,13 @@ public final class Cluster {
}
/**
- * Adds an address representing a group of Gremlin Servers, which will be added to the list of servers a
- * {@link Client} will try to contact to send requests to. The address should be parseable by
- * {@code InetAddress#getAllByName(String)}. That's the only validation performed at this point.
- * No connection to the hosts is attempted.
- * <p/>
- * Note that if the address argument is a hostname that resolves to multiple {@code INetAddress} objects that
- * happen to point to the same physical Gremlin Server instance the driver will think that it has more hosts
- * for it to send requests to than there are in reality. A common situation in which this issue may occur is
- * when IPv6 addressing is present as {@code InetAddress#getAllByName(String)} will return both the IPv6 and
- * the IPv4 addresses. In these cases, the issue can be resolved by simply using the IP address directly or
- * setting the Java system property {@code -Djava.net.preferIPv4Stack=true}.
+ * Adds the address of a Gremlin Server to the list of servers a {@link Client} will try to contact to send
+ * requests to. The address should be parseable by {@link InetAddress#getByName(String)}. That's the only
+ * validation performed at this point. No connection to the host is attempted.
*/
public Builder addContactPoint(final String address) {
try {
- this.addresses.addAll(Arrays.asList(InetAddress.getAllByName(address)));
+ this.addresses.add(InetAddress.getByName(address));
return this;
} catch (UnknownHostException e) {
throw new IllegalArgumentException(e.getMessage());
@@ -923,10 +914,9 @@ public final class Cluster {
}
/**
- * Add one or more addresses representing groups of Gremlin Servers, which will be added to the list of
- * servers a {@link Client} will try to contact to send requests to. The address should be parseable by
- * {@link InetAddress#getByName(String)}. That's the only validation performed at this point.
- * No connection to the host is attempted.
+ * Add one or more the addresses of a Gremlin Servers to the list of servers a {@link Client} will try to
+ * contact to send requests to. The address should be parseable by {@link InetAddress#getByName(String)}.
+ * That's the only validation performed at this point. No connection to the host is attempted.
*/
public Builder addContactPoints(final String... addresses) {
for (String address : addresses)
diff --git a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Host.java b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Host.java
index 5fd2b40..e225365 100644
--- a/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Host.java
+++ b/gremlin-driver/src/main/java/org/apache/tinkerpop/gremlin/driver/Host.java
@@ -46,7 +46,6 @@ public final class Host {
ScheduledFuture<?> retryThread = null;
Host(final InetSocketAddress address, final Cluster cluster) {
- assert(!address.isUnresolved());
this.cluster = cluster;
this.address = address;
this.hostUri = makeUriFromAddress(address, cluster.getPath(), cluster.connectionPoolSettings().enableSsl);
@@ -98,7 +97,7 @@ public final class Host {
private static URI makeUriFromAddress(final InetSocketAddress addy, final String path, final boolean ssl) {
try {
final String scheme = ssl ? "wss" : "ws";
- return new URI(scheme, null, addy.getAddress().getHostAddress(), addy.getPort(), path, null, null);
+ return new URI(scheme, null, addy.getHostName(), addy.getPort(), path, null, null);
} catch (URISyntaxException use) {
throw new RuntimeException(String.format("URI for host could not be constructed from: %s", addy), use);
}
diff --git a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/HostTest.java b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/HostTest.java
index 321e843..2c20261 100644
--- a/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/HostTest.java
+++ b/gremlin-driver/src/test/java/org/apache/tinkerpop/gremlin/driver/HostTest.java
@@ -19,26 +19,15 @@
package org.apache.tinkerpop.gremlin.driver;
import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.powermock.api.mockito.PowerMockito;
-import org.powermock.core.classloader.annotations.PrepareForTest;
-import org.powermock.modules.junit4.PowerMockRunner;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
-import java.net.UnknownHostException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Set;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* @author Stephen Mallette (http://stephen.genoprime.com)
*/
-@RunWith(PowerMockRunner.class)
public class HostTest {
@Test
@@ -46,7 +35,7 @@ public class HostTest {
final InetSocketAddress addy = new InetSocketAddress("localhost", 8182);
final Host host = new Host(addy, Cluster.open());
final URI webSocketUri = host.getHostUri();
- assertEquals("ws://" + addy.getAddress().getHostAddress() + ":8182/gremlin", webSocketUri.toString());
+ assertEquals("ws://localhost:8182/gremlin", webSocketUri.toString());
}
@Test
@@ -54,38 +43,7 @@ public class HostTest {
final InetSocketAddress addy = new InetSocketAddress("localhost", 8183);
final Host host = new Host(addy, Cluster.build().port(8183).path("/argh").create());
final URI webSocketUri = host.getHostUri();
- assertEquals("ws://" + addy.getAddress().getHostAddress() + ":8183/argh", webSocketUri.toString());
+ assertEquals("ws://localhost:8183/argh", webSocketUri.toString());
}
- @Test
- @PrepareForTest(Cluster.Builder.class)
- public void shouldFindAllHostsWithDeterministicAddress() throws Exception {
- final String hostname = "test.tinkerpop.apache.org";
- final Set<String> addresses = new HashSet<>(Arrays.asList("10.10.0.1", "10.10.0.2", "10.10.0.3"));
- final InetAddress[] hosts = addresses.stream().map(addr -> inetAddress(hostname, addr)).toArray(InetAddress[]::new);
-
- PowerMockito.mockStatic(InetAddress.class);
- PowerMockito.when(InetAddress.getAllByName(hostname)).thenReturn(hosts);
-
- final Cluster cluster = Cluster.build().addContactPoint(hostname).create();
-
- cluster.init();
-
- assertEquals(addresses.size(), cluster.allHosts().size());
-
- cluster.allHosts().forEach(host -> {
- final String uriHost = host.getHostUri().getHost();
- final String address = host.getAddress().getAddress().getHostAddress();
- assertEquals(address, uriHost);
- assertTrue(addresses.contains(address));
- });
- }
-
- private InetAddress inetAddress(final String hostname, final String address) {
- try {
- return InetAddress.getByAddress(hostname, InetAddress.getByName(address).getAddress());
- } catch (UnknownHostException ex) {
- throw new RuntimeException(ex);
- }
- }
}
[tinkerpop] 05/05: TINKERPOP-2076 Fixed bad merge operation in the
MemoryAccumulator
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit a3fd3d487c3c3759ee8f960abb4360fccf170de5
Author: stephen <sp...@gmail.com>
AuthorDate: Tue Nov 12 05:06:58 2019 -0500
TINKERPOP-2076 Fixed bad merge operation in the MemoryAccumulator
---
.../tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
index cc7b8de..fa72eb0 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
@@ -60,7 +60,7 @@ public final class MemoryAccumulator<A> extends AccumulatorV2<ObjectWritable<A>,
public void add(final ObjectWritable<A> v) {
if (this.value.isEmpty())
this.value = v;
- if (!v.isEmpty())
+ else if (!v.isEmpty())
this.value = new ObjectWritable<>(this.memoryComputeKey.getReducer().apply(value.get(), v.get()));
}
[tinkerpop] 03/05: TINKERPOP-2076 Bump to spark 3.0 with jdk11
support
Posted by sp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-2076
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 6833c7b89d27219476f0d4ee390f9f6182e65083
Author: stephen <sp...@gmail.com>
AuthorDate: Tue Nov 12 04:13:49 2019 -0500
TINKERPOP-2076 Bump to spark 3.0 with jdk11 support
---
CHANGELOG.asciidoc | 3 +-
gremlin-groovy/pom.xml | 2 +-
pom.xml | 2 +-
spark-gremlin/pom.xml | 254 ++++-----------------
.../spark/process/computer/MemoryAccumulator.java | 47 ++--
.../spark/process/computer/SparkMemory.java | 16 +-
6 files changed, 93 insertions(+), 231 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 45d4431..bae9f2b 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -32,7 +32,8 @@ This release also includes changes from <<release-3-4-3, 3.4.3>>.
* Renamed `SERVER_ERROR_SCRIPT_EVALUATION` to `SERVER_ERROR_EVALUATION` given that this response code applies to remote traversals as well as scripts.
* Configured GraphBinary as the default binary serialization format for the Java Driver.
* Configured GraphSON 3.0 as the default text serialization format when no serializer can be determined.
-* Upgraded to Neo4j 3.4.11.
+* Bumped to Neo4j 3.4.11.
+* Bumped to Spark 3.0.0.
* Added a parameterized `TypeTranslator` for use with `GroovyTranslator` that should produce more cache hits.
* Added support for `TextP` in Neo4j using its string search functions.
* Changed `TraversalStrategy` application methodology to apply each strategy in turn to each level of the traversal hierarchy starting from root down to children.
diff --git a/gremlin-groovy/pom.xml b/gremlin-groovy/pom.xml
index c4373b4..b890a99 100644
--- a/gremlin-groovy/pom.xml
+++ b/gremlin-groovy/pom.xml
@@ -34,7 +34,7 @@ limitations under the License.
<dependency>
<groupId>org.apache.ivy</groupId>
<artifactId>ivy</artifactId>
- <version>2.3.0</version>
+ <version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.groovy</groupId>
diff --git a/pom.xml b/pom.xml
index 444a4e3..7f14fd7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -159,7 +159,7 @@ limitations under the License.
<netty.version>4.1.42.Final</netty.version>
<slf4j.version>1.7.25</slf4j.version>
<snakeyaml.version>1.15</snakeyaml.version>
- <spark.version>2.4.0</spark.version>
+ <spark.version>3.0.0-preview</spark.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml
index 695d743..0d7a64d 100644
--- a/spark-gremlin/pom.xml
+++ b/spark-gremlin/pom.xml
@@ -32,7 +32,7 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <version>14.0.1</version>
+ <version>16.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
@@ -44,257 +44,105 @@
<artifactId>hadoop-gremlin</artifactId>
<version>${project.version}</version>
<exclusions>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>javax.servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-net</groupId>
- <artifactId>commons-net</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-io</groupId>
- <artifactId>commons-io</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
+ <!-- use our snappy as there is conflict within spark-->
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
+ <!-- use spark's avro -->
<exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
+ <!-- use spark's math -->
<exclusion>
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</exclusion>
+ <!-- use spark's netty 4-->
<exclusion>
<groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
+ <artifactId>netty-all</artifactId>
</exclusion>
+ <!-- use spark's activation -->
+ <exclusion>
+ <groupId>javax.activation</groupId>
+ <artifactId>activation</artifactId>
+ </exclusion>
+ <!-- use zookeeper's netty 3 -->
<exclusion>
<groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
+ <artifactId>netty</artifactId>
</exclusion>
+ <!-- use sparks commons-compress -->
<exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- SPARK -->
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.11</artifactId>
+ <artifactId>spark-core_2.12</artifactId>
<version>${spark.version}</version>
<exclusions>
- <!-- self conflicts -->
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_2.11</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.activation</groupId>
- <artifactId>activation</artifactId>
- </exclusion>
<exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
<exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
</exclusion>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy</artifactId>
</exclusion>
<exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
</exclusion>
<exclusion>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- </exclusion>
- <exclusion>
- <groupId>javax.servlet</groupId>
- <artifactId>servlet-api</artifactId>
- </exclusion>
- <!-- gremlin-core conflicts -->
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>jcl-over-slf4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.ivy</groupId>
- <artifactId>ivy</artifactId>
- </exclusion>
- <!-- gremlin-groovy conflicts -->
- <exclusion>
- <groupId>jline</groupId>
- <artifactId>jline</artifactId>
- </exclusion>
- <!-- hadoop conflicts -->
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-client</artifactId>
- </exclusion>
- <!-- lgpl conflicts -->
- <exclusion>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </exclusion>
- <!-- avro conflicts -->
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- </exclusion>
</exclusions>
</dependency>
- <!-- consistent dependencies -->
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>2.11.8</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_2.11</artifactId>
- <version>1.0.5</version>
- <exclusions>
- <exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>2.6.7</version>
- </dependency>
- <dependency>
- <groupId>commons-lang</groupId>
- <artifactId>commons-lang</artifactId>
- <version>${commons.lang.version}</version>
- </dependency>
- <dependency>
- <groupId>com.thoughtworks.paranamer</groupId>
- <artifactId>paranamer</artifactId>
- <version>2.6</version>
- </dependency>
+ <!-- spark self-conflict and hadoop conflict -->
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
- <version>1.1.1.7</version>
+ <version>1.1.7.3</version>
</dependency>
+ <!-- spark self-conflict -->
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- <version>4.1.32.Final</version>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>2.12.10</version>
</dependency>
+ <!-- spark self-confict -->
<dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>3.9.9.Final</version>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ <version>2.10.0</version>
</dependency>
+ <!-- spark self-confict -->
<dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-compress</artifactId>
- <version>1.19</version>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.6</version>
+ <exclusions>
+ <!-- use gremlin-groovy's jline -->
+ <exclusion>
+ <groupId>jline</groupId>
+ <artifactId>jline</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<!-- TEST -->
<dependency>
@@ -302,16 +150,6 @@
<artifactId>gremlin-test</artifactId>
<version>${project.version}</version>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.objenesis</groupId>
- <artifactId>objenesis</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.tinkerpop</groupId>
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
index cf8cb25..cc7b8de 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/MemoryAccumulator.java
@@ -19,37 +19,58 @@
package org.apache.tinkerpop.gremlin.spark.process.computer;
-import org.apache.spark.AccumulatorParam;
+import org.apache.spark.util.AccumulatorV2;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.process.computer.MemoryComputeKey;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
*/
-public final class MemoryAccumulator<A> implements AccumulatorParam<ObjectWritable<A>> {
+public final class MemoryAccumulator<A> extends AccumulatorV2<ObjectWritable<A>, ObjectWritable<A>> {
private final MemoryComputeKey<A> memoryComputeKey;
+ private ObjectWritable<A> value;
- public MemoryAccumulator(final MemoryComputeKey<A> memoryComputeKey) {
+ MemoryAccumulator(final MemoryComputeKey<A> memoryComputeKey) {
+ this(memoryComputeKey, ObjectWritable.empty());
+ }
+
+ private MemoryAccumulator(final MemoryComputeKey<A> memoryComputeKey, final ObjectWritable<A> initial) {
this.memoryComputeKey = memoryComputeKey;
+ this.value = initial;
+ }
+
+ @Override
+ public boolean isZero() {
+ return ObjectWritable.empty().equals(value);
+ }
+
+ @Override
+ public AccumulatorV2<ObjectWritable<A>, ObjectWritable<A>> copy() {
+ return new MemoryAccumulator<>(this.memoryComputeKey, this.value);
+ }
+
+ @Override
+ public void reset() {
+ this.value = ObjectWritable.empty();
}
@Override
- public ObjectWritable<A> addAccumulator(final ObjectWritable<A> a, final ObjectWritable<A> b) {
- if (a.isEmpty())
- return b;
- if (b.isEmpty())
- return a;
- return new ObjectWritable<>(this.memoryComputeKey.getReducer().apply(a.get(), b.get()));
+ public void add(final ObjectWritable<A> v) {
+ if (this.value.isEmpty())
+ this.value = v;
+ if (!v.isEmpty())
+ this.value = new ObjectWritable<>(this.memoryComputeKey.getReducer().apply(value.get(), v.get()));
}
@Override
- public ObjectWritable<A> addInPlace(final ObjectWritable<A> a, final ObjectWritable<A> b) {
- return this.addAccumulator(a, b);
+ public void merge(final AccumulatorV2<ObjectWritable<A>, ObjectWritable<A>> other) {
+ this.add(other.value());
}
@Override
- public ObjectWritable<A> zero(final ObjectWritable<A> a) {
- return ObjectWritable.empty();
+ public ObjectWritable<A> value() {
+ return this.value;
}
}
diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
index bf8590e..5a04162 100644
--- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
+++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkMemory.java
@@ -18,7 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.spark.process.computer;
-import org.apache.spark.Accumulator;
+import org.apache.spark.util.AccumulatorV2;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.broadcast.Broadcast;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
@@ -46,7 +46,7 @@ import java.util.concurrent.atomic.AtomicLong;
public final class SparkMemory implements Memory.Admin, Serializable {
public final Map<String, MemoryComputeKey> memoryComputeKeys = new HashMap<>();
- private final Map<String, Accumulator<ObjectWritable>> sparkMemory = new HashMap<>();
+ private final Map<String, AccumulatorV2<ObjectWritable,ObjectWritable>> sparkMemory = new HashMap<>();
private final AtomicInteger iteration = new AtomicInteger(0);
private final AtomicLong runtime = new AtomicLong(0l);
private Broadcast<Map<String, Object>> broadcast;
@@ -62,9 +62,9 @@ public final class SparkMemory implements Memory.Admin, Serializable {
this.memoryComputeKeys.put(mapReduce.getMemoryKey(), MemoryComputeKey.of(mapReduce.getMemoryKey(), Operator.assign, false, false));
}
for (final MemoryComputeKey memoryComputeKey : this.memoryComputeKeys.values()) {
- this.sparkMemory.put(
- memoryComputeKey.getKey(),
- sparkContext.accumulator(ObjectWritable.empty(), memoryComputeKey.getKey(), new MemoryAccumulator<>(memoryComputeKey)));
+ final AccumulatorV2<ObjectWritable, ObjectWritable> accumulator = new MemoryAccumulator<>(memoryComputeKey);
+ JavaSparkContext.toSparkContext(sparkContext).register(accumulator, memoryComputeKey.getKey());
+ this.sparkMemory.put(memoryComputeKey.getKey(), accumulator);
}
this.broadcast = sparkContext.broadcast(Collections.emptyMap());
}
@@ -135,8 +135,10 @@ public final class SparkMemory implements Memory.Admin, Serializable {
checkKeyValue(key, value);
if (this.inExecute)
throw Memory.Exceptions.memorySetOnlyDuringVertexProgramSetUpAndTerminate(key);
- else
- this.sparkMemory.get(key).setValue(new ObjectWritable<>(value));
+ else {
+ this.sparkMemory.get(key).reset();
+ this.sparkMemory.get(key).add(new ObjectWritable<>(value));
+ }
}
@Override