You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ar...@apache.org on 2020/08/13 12:00:59 UTC
[beam] branch master updated: [BEAM-10694] Work around
serialization issue with ReaderContext by memoizing the serialized form and
propagating it forward. (#12556)
This is an automated email from the ASF dual-hosted git repository.
aromanenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2def4cc [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward. (#12556)
2def4cc is described below
commit 2def4cc32188e8ea1128e72bce57e99c398a6bd2
Author: Lukasz Cwik <lu...@gmail.com>
AuthorDate: Thu Aug 13 04:59:36 2020 -0700
[BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward. (#12556)
* [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward.
* fixup! Address spotbugs failure
---
.../apache/beam/sdk/io/hcatalog/HCatalogIO.java | 49 +++++++++++++++++++++-
.../beam/sdk/io/hcatalog/HCatalogIOTest.java | 26 ++++++++++++
2 files changed, 73 insertions(+), 2 deletions(-)
diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
index 0c09c89..67c0af9 100644
--- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
@@ -20,6 +20,8 @@ package org.apache.beam.sdk.io.hcatalog;
import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
import com.google.auto.value.AutoValue;
+import java.io.IOException;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -39,6 +41,7 @@ import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Watch;
import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.values.PBegin;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.sdk.values.PDone;
@@ -158,7 +161,15 @@ public class HCatalogIO {
abstract @Nullable String getFilter();
- abstract @Nullable ReaderContext getContext();
+ @Nullable
+ ReaderContext getContext() {
+ if (getContextHolder() == null) {
+ return null;
+ }
+ return getContextHolder().get();
+ }
+
+ abstract @Nullable ReaderContextHolder getContextHolder();
abstract @Nullable Integer getSplitId();
@@ -172,6 +183,7 @@ public class HCatalogIO {
@AutoValue.Builder
abstract static class Builder {
+
abstract Builder setConfigProperties(Map<String, String> configProperties);
abstract Builder setDatabase(String database);
@@ -182,7 +194,11 @@ public class HCatalogIO {
abstract Builder setSplitId(Integer splitId);
- abstract Builder setContext(ReaderContext context);
+ abstract Builder setContextHolder(ReaderContextHolder context);
+
+ Builder setContext(ReaderContext context) {
+ return this.setContextHolder(new ReaderContextHolder(context));
+ }
abstract Builder setPollingInterval(Duration pollingInterval);
@@ -276,6 +292,35 @@ public class HCatalogIO {
builder.addIfNotNull(DisplayData.item("database", getDatabase()));
builder.addIfNotNull(DisplayData.item("filter", getFilter()));
}
+
+ /**
+ * We specifically use a holder which replaces the implementation of what is being serialized to
+ * cache the serialized version instead of re-serializing the ReaderContext. See BEAM-10694 for
+ * additional details.
+ */
+ static class ReaderContextHolder implements Serializable {
+
+ private final byte[] serializedReaderContext;
+ private transient ReaderContext readerContext;
+
+ public ReaderContextHolder(ReaderContext readerContext) {
+ this.serializedReaderContext = SerializableUtils.serializeToByteArray(readerContext);
+ this.readerContext = readerContext;
+ }
+
+ private ReaderContext get() {
+ return readerContext;
+ }
+
+ private void readObject(java.io.ObjectInputStream in)
+ throws IOException, ClassNotFoundException {
+ in.defaultReadObject();
+ readerContext =
+ (ReaderContext)
+ SerializableUtils.deserializeFromByteArray(
+ serializedReaderContext, "ReaderContext");
+ }
+ }
}
/** A HCatalog {@link BoundedSource} reading {@link HCatRecord} from a given instance. */
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
index 786984b..c7f3a2e 100644
--- a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
@@ -57,6 +57,7 @@ import org.apache.beam.sdk.transforms.Create;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Watch;
+import org.apache.beam.sdk.util.SerializableUtils;
import org.apache.beam.sdk.util.UserCodeException;
import org.apache.beam.sdk.values.PCollection;
import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
@@ -292,6 +293,31 @@ public class HCatalogIOTest implements Serializable {
.expand(null);
}
+ /** Regression test for BEAM-10694. */
+ @Test
+ public void testReadTransformCanBeSerializedMultipleTimes() throws Exception {
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE);
+ SerializableUtils.clone(SerializableUtils.clone(spec));
+ }
+
+ /** Regression test for BEAM-10694. */
+ @Test
+ public void testSourceCanBeSerializedMultipleTimes() throws Exception {
+ ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+ HCatalogIO.Read spec =
+ HCatalogIO.read()
+ .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+ .withContext(context)
+ .withTable(TEST_TABLE);
+ BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
+ SerializableUtils.clone(SerializableUtils.clone(source));
+ }
+
/** Test of Read using SourceTestUtils.readFromSource(..). */
@Test
@NeedsTestData