You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sr...@apache.org on 2018/11/12 17:00:37 UTC
[flink] branch release-1.6 updated (4b0a941 -> fffe6e1)
This is an automated email from the ASF dual-hosted git repository.
srichter pushed a change to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 4b0a941 [FLINK-10821] E2E now uses externalized checkpoint
new 8196a69 [FLINK-10839][serializer] Fix implementation of PojoSerializer.duplicate() w.r.t. subclass serializer
new fffe6e1 [FLINK-10827][tests] Add test for duplicate() to SerializerTestBase
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../api/java/typeutils/runtime/PojoSerializer.java | 34 +++---
.../api/common/typeutils/SerializerTestBase.java | 130 ++++++++++++++++++---
2 files changed, 131 insertions(+), 33 deletions(-)
[flink] 01/02: [FLINK-10839][serializer] Fix implementation of
PojoSerializer.duplicate() w.r.t. subclass serializer
Posted by sr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
srichter pushed a commit to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 8196a69f95af08ec829c77e190161d99f186aa57
Author: Stefan Richter <s....@data-artisans.com>
AuthorDate: Fri Nov 9 12:56:02 2018 +0100
[FLINK-10839][serializer] Fix implementation of PojoSerializer.duplicate() w.r.t. subclass serializer
---
.../api/java/typeutils/runtime/PojoSerializer.java | 36 ++++++++++++----------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index a4d086d..f38a620 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -18,18 +18,6 @@
package org.apache.flink.api.java.typeutils.runtime;
-import java.io.IOException;
-import java.io.ObjectInputStream;
-import java.io.ObjectOutputStream;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.Map;
-import java.util.Objects;
-
import org.apache.flink.annotation.Internal;
import org.apache.flink.annotation.VisibleForTesting;
import org.apache.flink.api.common.ExecutionConfig;
@@ -39,6 +27,8 @@ import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
+import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil;
import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
@@ -50,6 +40,18 @@ import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.util.Preconditions;
+import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.lang.reflect.Field;
+import java.lang.reflect.Modifier;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+import java.util.Objects;
+
import static org.apache.flink.util.Preconditions.checkNotNull;
@Internal
@@ -179,11 +181,13 @@ public final class PojoSerializer<T> extends TypeSerializer<T> {
}
}
- if (stateful) {
- return new PojoSerializer<T>(clazz, duplicateFieldSerializers, fields, executionConfig);
- } else {
- return this;
+ if (!stateful) {
+ // as a small memory optimization, we can share the same object between instances
+ duplicateFieldSerializers = fieldSerializers;
}
+
+ // we must create a new instance, otherwise the subclassSerializerCache can create concurrency problems
+ return new PojoSerializer<>(clazz, duplicateFieldSerializers, fields, executionConfig);
}
[flink] 02/02: [FLINK-10827][tests] Add test for duplicate() to
SerializerTestBase
Posted by sr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
srichter pushed a commit to branch release-1.6
in repository https://gitbox.apache.org/repos/asf/flink.git
commit fffe6e18bec2361001c8b852aec465d9fe4fc718
Author: Stefan Richter <s....@data-artisans.com>
AuthorDate: Thu Nov 8 14:44:08 2018 +0100
[FLINK-10827][tests] Add test for duplicate() to SerializerTestBase
This closes #7061.
---
.../api/java/typeutils/runtime/PojoSerializer.java | 2 -
.../api/common/typeutils/SerializerTestBase.java | 130 ++++++++++++++++++---
2 files changed, 113 insertions(+), 19 deletions(-)
diff --git a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
index f38a620..021bc24 100644
--- a/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
+++ b/flink-core/src/main/java/org/apache/flink/api/java/typeutils/runtime/PojoSerializer.java
@@ -27,8 +27,6 @@ import org.apache.flink.api.common.typeutils.GenericTypeSerializerConfigSnapshot
import org.apache.flink.api.common.typeutils.TypeSerializer;
import org.apache.flink.api.common.typeutils.TypeSerializerConfigSnapshot;
import org.apache.flink.api.common.typeutils.TypeSerializerSerializationUtil;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshot;
-import org.apache.flink.api.common.typeutils.TypeSerializerSnapshotSerializationUtil;
import org.apache.flink.api.common.typeutils.UnloadableDummyTypeSerializer;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.api.java.typeutils.TypeExtractor;
diff --git a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
index 1997866..61aee94 100644
--- a/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
+++ b/flink-core/src/test/java/org/apache/flink/api/common/typeutils/SerializerTestBase.java
@@ -18,33 +18,40 @@
package org.apache.flink.api.common.typeutils;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-import java.util.Arrays;
-
import org.apache.flink.api.java.typeutils.runtime.NullableSerializer;
+import org.apache.flink.core.memory.DataInputDeserializer;
+import org.apache.flink.core.memory.DataInputView;
import org.apache.flink.core.memory.DataInputViewStreamWrapper;
+import org.apache.flink.core.memory.DataOutputSerializer;
+import org.apache.flink.core.memory.DataOutputView;
import org.apache.flink.core.memory.DataOutputViewStreamWrapper;
import org.apache.flink.util.InstantiationUtil;
+import org.apache.flink.util.Preconditions;
import org.apache.flink.util.TestLogger;
-import org.junit.Assert;
import org.apache.commons.lang3.SerializationException;
import org.apache.commons.lang3.SerializationUtils;
-import org.apache.flink.core.memory.DataInputView;
-import org.apache.flink.core.memory.DataOutputView;
+import org.junit.Assert;
import org.junit.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.concurrent.CyclicBarrier;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
/**
* Abstract test base for serializers.
*
@@ -437,6 +444,33 @@ public abstract class SerializerTestBase<T> extends TestLogger {
}
}
+ @Test
+ public void testDuplicate() throws Exception {
+ final int numThreads = 10;
+ final TypeSerializer<T> serializer = getSerializer();
+ final CyclicBarrier startLatch = new CyclicBarrier(numThreads);
+ final List<SerializerRunner<T>> concurrentRunners = new ArrayList<>(numThreads);
+ Assert.assertEquals(serializer, serializer.duplicate());
+
+ T[] testData = getData();
+
+ for (int i = 0; i < numThreads; ++i) {
+ SerializerRunner<T> runner = new SerializerRunner<>(
+ startLatch,
+ serializer.duplicate(),
+ testData,
+ 120L);
+
+ runner.start();
+ concurrentRunners.add(runner);
+ }
+
+ for (SerializerRunner<T> concurrentRunner : concurrentRunners) {
+ concurrentRunner.join();
+ concurrentRunner.checkResult();
+ }
+ }
+
// --------------------------------------------------------------------------------------------
protected void deepEquals(String message, T should, T is) {
@@ -527,6 +561,68 @@ public abstract class SerializerTestBase<T> extends TestLogger {
}
}
+ /**
+ * Runner to test serializer duplication via concurrency.
+ * @param <T> type of the test elements.
+ */
+ static class SerializerRunner<T> extends Thread {
+ final CyclicBarrier allReadyBarrier;
+ final TypeSerializer<T> serializer;
+ final T[] testData;
+ final long durationLimitMillis;
+ Exception failure;
+
+ SerializerRunner(
+ CyclicBarrier allReadyBarrier,
+ TypeSerializer<T> serializer,
+ T[] testData,
+ long testTargetDurationMillis) {
+
+ this.allReadyBarrier = allReadyBarrier;
+ this.serializer = serializer;
+ this.testData = testData;
+ this.durationLimitMillis = testTargetDurationMillis;
+ this.failure = null;
+ }
+
+ @Override
+ public void run() {
+ DataInputDeserializer dataInputDeserializer = new DataInputDeserializer();
+ DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
+ try {
+ allReadyBarrier.await();
+ final long endTimeNanos = System.nanoTime() + durationLimitMillis * 1_000_000L;
+ while (true) {
+ for (T testItem : testData) {
+ serializer.serialize(testItem, dataOutputSerializer);
+ dataInputDeserializer.setBuffer(
+ dataOutputSerializer.getSharedBuffer(),
+ 0,
+ dataOutputSerializer.length());
+ T serdeTestItem = serializer.deserialize(dataInputDeserializer);
+ T copySerdeTestItem = serializer.copy(serdeTestItem);
+ dataOutputSerializer.clear();
+
+ Preconditions.checkState(Objects.deepEquals(testItem, copySerdeTestItem),
+ "Serialization/Deserialization cycle resulted in an object that are not equal to the original.");
+
+ // try to enforce some upper bound to the test time
+ if (System.nanoTime() >= endTimeNanos) {
+ return;
+ }
+ }
+ }
+ } catch (Exception ex) {
+ failure = ex;
+ }
+ }
+
+ void checkResult() throws Exception {
+ if (failure != null) {
+ throw failure;
+ }
+ }
+ }
private static final class TestInputView extends DataInputStream implements DataInputView {