You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ar...@apache.org on 2020/08/13 12:00:59 UTC

[beam] branch master updated: [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward. (#12556)

This is an automated email from the ASF dual-hosted git repository.

aromanenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 2def4cc  [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward. (#12556)
2def4cc is described below

commit 2def4cc32188e8ea1128e72bce57e99c398a6bd2
Author: Lukasz Cwik <lu...@gmail.com>
AuthorDate: Thu Aug 13 04:59:36 2020 -0700

    [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward. (#12556)
    
    * [BEAM-10694] Work around serialization issue with ReaderContext by memoizing the serialized form and propagating it forward.
    
    * fixup! Address spotbugs failure
---
 .../apache/beam/sdk/io/hcatalog/HCatalogIO.java    | 49 +++++++++++++++++++++-
 .../beam/sdk/io/hcatalog/HCatalogIOTest.java       | 26 ++++++++++++
 2 files changed, 73 insertions(+), 2 deletions(-)

diff --git a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
index 0c09c89..67c0af9 100644
--- a/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
+++ b/sdks/java/io/hcatalog/src/main/java/org/apache/beam/sdk/io/hcatalog/HCatalogIO.java
@@ -20,6 +20,8 @@ package org.apache.beam.sdk.io.hcatalog;
 import static org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkArgument;
 
 import com.google.auto.value.AutoValue;
+import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -39,6 +41,7 @@ import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Watch;
 import org.apache.beam.sdk.transforms.Watch.Growth.TerminationCondition;
 import org.apache.beam.sdk.transforms.display.DisplayData;
+import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.sdk.values.PDone;
@@ -158,7 +161,15 @@ public class HCatalogIO {
 
     abstract @Nullable String getFilter();
 
-    abstract @Nullable ReaderContext getContext();
+    @Nullable
+    ReaderContext getContext() {
+      if (getContextHolder() == null) {
+        return null;
+      }
+      return getContextHolder().get();
+    }
+
+    abstract @Nullable ReaderContextHolder getContextHolder();
 
     abstract @Nullable Integer getSplitId();
 
@@ -172,6 +183,7 @@ public class HCatalogIO {
 
     @AutoValue.Builder
     abstract static class Builder {
+
       abstract Builder setConfigProperties(Map<String, String> configProperties);
 
       abstract Builder setDatabase(String database);
@@ -182,7 +194,11 @@ public class HCatalogIO {
 
       abstract Builder setSplitId(Integer splitId);
 
-      abstract Builder setContext(ReaderContext context);
+      abstract Builder setContextHolder(ReaderContextHolder context);
+
+      Builder setContext(ReaderContext context) {
+        return this.setContextHolder(new ReaderContextHolder(context));
+      }
 
       abstract Builder setPollingInterval(Duration pollingInterval);
 
@@ -276,6 +292,35 @@ public class HCatalogIO {
       builder.addIfNotNull(DisplayData.item("database", getDatabase()));
       builder.addIfNotNull(DisplayData.item("filter", getFilter()));
     }
+
+    /**
+     * We specifically use a holder which replaces the implementation of what is being serialized to
+     * cache the serialized version instead of re-serializing the ReaderContext. See BEAM-10694 for
+     * additional details.
+     */
+    static class ReaderContextHolder implements Serializable {
+
+      private final byte[] serializedReaderContext;
+      private transient ReaderContext readerContext;
+
+      public ReaderContextHolder(ReaderContext readerContext) {
+        this.serializedReaderContext = SerializableUtils.serializeToByteArray(readerContext);
+        this.readerContext = readerContext;
+      }
+
+      private ReaderContext get() {
+        return readerContext;
+      }
+
+      private void readObject(java.io.ObjectInputStream in)
+          throws IOException, ClassNotFoundException {
+        in.defaultReadObject();
+        readerContext =
+            (ReaderContext)
+                SerializableUtils.deserializeFromByteArray(
+                    serializedReaderContext, "ReaderContext");
+      }
+    }
   }
 
   /** A HCatalog {@link BoundedSource} reading {@link HCatRecord} from a given instance. */
diff --git a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
index 786984b..c7f3a2e 100644
--- a/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
+++ b/sdks/java/io/hcatalog/src/test/java/org/apache/beam/sdk/io/hcatalog/HCatalogIOTest.java
@@ -57,6 +57,7 @@ import org.apache.beam.sdk.transforms.Create;
 import org.apache.beam.sdk.transforms.DoFn;
 import org.apache.beam.sdk.transforms.ParDo;
 import org.apache.beam.sdk.transforms.Watch;
+import org.apache.beam.sdk.util.SerializableUtils;
 import org.apache.beam.sdk.util.UserCodeException;
 import org.apache.beam.sdk.values.PCollection;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
@@ -292,6 +293,31 @@ public class HCatalogIOTest implements Serializable {
         .expand(null);
   }
 
+  /** Regression test for BEAM-10694. */
+  @Test
+  public void testReadTransformCanBeSerializedMultipleTimes() throws Exception {
+    ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+    HCatalogIO.Read spec =
+        HCatalogIO.read()
+            .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+            .withContext(context)
+            .withTable(TEST_TABLE);
+    SerializableUtils.clone(SerializableUtils.clone(spec));
+  }
+
+  /** Regression test for BEAM-10694. */
+  @Test
+  public void testSourceCanBeSerializedMultipleTimes() throws Exception {
+    ReaderContext context = getReaderContext(getConfigPropertiesAsMap(service.getHiveConf()));
+    HCatalogIO.Read spec =
+        HCatalogIO.read()
+            .withConfigProperties(getConfigPropertiesAsMap(service.getHiveConf()))
+            .withContext(context)
+            .withTable(TEST_TABLE);
+    BoundedHCatalogSource source = new BoundedHCatalogSource(spec);
+    SerializableUtils.clone(SerializableUtils.clone(source));
+  }
+
   /** Test of Read using SourceTestUtils.readFromSource(..). */
   @Test
   @NeedsTestData