You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@beam.apache.org by Sahith Nallapareddy via dev <de...@beam.apache.org> on 2022/09/27 22:25:42 UTC
NullPointerException when getting timestamp from ProcessContext
Hello,
We have seen this error popup when running pipelines on the dataflow runner
and not sure exactly why. It seems like we get a null pointer exception
when accessing the timestamp via the ProcessContext. This pipeline is using
runner_v2 and is a cross transform pipeline so that may be why. Not sure
what we can do here anyone seen this before? Here is the exception in full:
java.lang.NullPointerException at
org.apache.beam.fn.harness.FnApiDoFnRunner$ProcessBundleContextBase.timestamp(FnApiDoFnRunner.java:2496)
at
com.spotify.scio.transforms.BaseAsyncLookupDoFn.lambda$processElement$7(BaseAsyncLookupDoFn.java:203)
at
com.spotify.scio.transforms.FutureHandlers$Guava$1.onSuccess(FutureHandlers.java:66)
at
com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
at
com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
at
com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
at
com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
at
com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
at
com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:57)
at
com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:27)
at
java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859)
at
java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837)
at
java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
at
java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073)
at
com.spotify.futures.ListenableToCompletableFutureWrapper.onSuccess(ListenableToCompletableFutureWrapper.java:50)
at
com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
at
com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
at
com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
at
com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
at
com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
at
com.google.common.util.concurrent.AbstractTransformFuture$TransformFuture.setResult(AbstractTransformFuture.java:255)
at
com.google.common.util.concurrent.AbstractTransformFuture.run(AbstractTransformFuture.java:167)
at
com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
at
com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
at
com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
at
com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
at
com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation$GrpcFuture.set(AbstractRetryingOperation.java:92)
at
com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onOK(RetryingReadRowsOperation.java:265)
at
com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation.onClose(AbstractRetryingOperation.java:174)
at
com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onClose(RetryingReadRowsOperation.java:231)
at
io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at
com.google.cloud.bigtable.grpc.io.Watchdog$WatchedCall$1.onClose(Watchdog.java:188)
at
com.google.cloud.bigtable.grpc.io.ChannelPool$InstrumentedChannel$2.onClose(ChannelPool.java:213)
at
io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at
io.grpc.census.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:802)
at
io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
at
io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
at
io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
at
io.grpc.census.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:428)
at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:562)
at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:743)
at
io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:722)
at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at
io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
Re: NullPointerException when getting timestamp from ProcessContext
Posted by Sahith Nallapareddy via dev <de...@beam.apache.org>.
ah thank you Claire! Sounds good for us
Thanks,
Sahith
On Wed, Sep 28, 2022 at 2:36 AM Claire McGinty <cl...@gmail.com>
wrote:
> Hey Sahith! That looks related this bug
> <https://github.com/spotify/scio/issues/4496>, which has been fixed in
> Scio by upgrading to the newer DoFn#processElement signature, and will be
> available in the upcoming 0.12.0 release :)
>
> Best,
> Claire
>
> On Wed, Sep 28, 2022 at 12:26 AM Sahith Nallapareddy via dev <
> dev@beam.apache.org> wrote:
>
>> Hello,
>>
>> We have seen this error popup when running pipelines on the dataflow
>> runner and not sure exactly why. It seems like we get a null pointer
>> exception when accessing the timestamp via the ProcessContext. This
>> pipeline is using runner_v2 and is a cross transform pipeline so that may
>> be why. Not sure what we can do here anyone seen this before? Here is the
>> exception in full:
>>
>> java.lang.NullPointerException at
>> org.apache.beam.fn.harness.FnApiDoFnRunner$ProcessBundleContextBase.timestamp(FnApiDoFnRunner.java:2496)
>> at
>> com.spotify.scio.transforms.BaseAsyncLookupDoFn.lambda$processElement$7(BaseAsyncLookupDoFn.java:203)
>> at
>> com.spotify.scio.transforms.FutureHandlers$Guava$1.onSuccess(FutureHandlers.java:66)
>> at
>> com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
>> at
>> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
>> at
>> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
>> at
>> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
>> at
>> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
>> at
>> com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:57)
>> at
>> com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:27)
>> at
>> java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859)
>> at
>> java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837)
>> at
>> java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
>> at
>> java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073)
>> at
>> com.spotify.futures.ListenableToCompletableFutureWrapper.onSuccess(ListenableToCompletableFutureWrapper.java:50)
>> at
>> com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
>> at
>> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
>> at
>> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
>> at
>> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
>> at
>> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
>> at
>> com.google.common.util.concurrent.AbstractTransformFuture$TransformFuture.setResult(AbstractTransformFuture.java:255)
>> at
>> com.google.common.util.concurrent.AbstractTransformFuture.run(AbstractTransformFuture.java:167)
>> at
>> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
>> at
>> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
>> at
>> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
>> at
>> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
>> at
>> com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation$GrpcFuture.set(AbstractRetryingOperation.java:92)
>> at
>> com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onOK(RetryingReadRowsOperation.java:265)
>> at
>> com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation.onClose(AbstractRetryingOperation.java:174)
>> at
>> com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onClose(RetryingReadRowsOperation.java:231)
>> at
>> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
>> at
>> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
>> at
>> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
>> at
>> com.google.cloud.bigtable.grpc.io.Watchdog$WatchedCall$1.onClose(Watchdog.java:188)
>> at
>> com.google.cloud.bigtable.grpc.io.ChannelPool$InstrumentedChannel$2.onClose(ChannelPool.java:213)
>> at
>> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
>> at
>> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
>> at
>> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
>> at
>> io.grpc.census.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:802)
>> at
>> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
>> at
>> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
>> at
>> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
>> at
>> io.grpc.census.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:428)
>> at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:562)
>> at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) at
>> io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:743)
>> at
>> io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:722)
>> at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at
>> io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) at
>> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
>> at
>> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
>> at java.base/java.lang.Thread.run(Thread.java:829)
>>
>
Re: NullPointerException when getting timestamp from ProcessContext
Posted by Claire McGinty <cl...@gmail.com>.
Hey Sahith! That looks related this bug
<https://github.com/spotify/scio/issues/4496>, which has been fixed in Scio
by upgrading to the newer DoFn#processElement signature, and will be
available in the upcoming 0.12.0 release :)
Best,
Claire
On Wed, Sep 28, 2022 at 12:26 AM Sahith Nallapareddy via dev <
dev@beam.apache.org> wrote:
> Hello,
>
> We have seen this error popup when running pipelines on the dataflow
> runner and not sure exactly why. It seems like we get a null pointer
> exception when accessing the timestamp via the ProcessContext. This
> pipeline is using runner_v2 and is a cross transform pipeline so that may
> be why. Not sure what we can do here anyone seen this before? Here is the
> exception in full:
>
> java.lang.NullPointerException at
> org.apache.beam.fn.harness.FnApiDoFnRunner$ProcessBundleContextBase.timestamp(FnApiDoFnRunner.java:2496)
> at
> com.spotify.scio.transforms.BaseAsyncLookupDoFn.lambda$processElement$7(BaseAsyncLookupDoFn.java:203)
> at
> com.spotify.scio.transforms.FutureHandlers$Guava$1.onSuccess(FutureHandlers.java:66)
> at
> com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
> at
> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
> at
> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
> at
> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
> at
> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
> at
> com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:57)
> at
> com.spotify.futures.CompletableToListenableFutureWrapper.accept(CompletableToListenableFutureWrapper.java:27)
> at
> java.base/java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:859)
> at
> java.base/java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:837)
> at
> java.base/java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:506)
> at
> java.base/java.util.concurrent.CompletableFuture.complete(CompletableFuture.java:2073)
> at
> com.spotify.futures.ListenableToCompletableFutureWrapper.onSuccess(ListenableToCompletableFutureWrapper.java:50)
> at
> com.google.common.util.concurrent.Futures$CallbackListener.run(Futures.java:1139)
> at
> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
> at
> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
> at
> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
> at
> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
> at
> com.google.common.util.concurrent.AbstractTransformFuture$TransformFuture.setResult(AbstractTransformFuture.java:255)
> at
> com.google.common.util.concurrent.AbstractTransformFuture.run(AbstractTransformFuture.java:167)
> at
> com.google.common.util.concurrent.DirectExecutor.execute(DirectExecutor.java:31)
> at
> com.google.common.util.concurrent.AbstractFuture.executeListener(AbstractFuture.java:1277)
> at
> com.google.common.util.concurrent.AbstractFuture.complete(AbstractFuture.java:1038)
> at
> com.google.common.util.concurrent.AbstractFuture.set(AbstractFuture.java:783)
> at
> com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation$GrpcFuture.set(AbstractRetryingOperation.java:92)
> at
> com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onOK(RetryingReadRowsOperation.java:265)
> at
> com.google.cloud.bigtable.grpc.async.AbstractRetryingOperation.onClose(AbstractRetryingOperation.java:174)
> at
> com.google.cloud.bigtable.grpc.scanner.RetryingReadRowsOperation.onClose(RetryingReadRowsOperation.java:231)
> at
> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
> at
> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
> at
> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
> at
> com.google.cloud.bigtable.grpc.io.Watchdog$WatchedCall$1.onClose(Watchdog.java:188)
> at
> com.google.cloud.bigtable.grpc.io.ChannelPool$InstrumentedChannel$2.onClose(ChannelPool.java:213)
> at
> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
> at
> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
> at
> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
> at
> io.grpc.census.CensusStatsModule$StatsClientInterceptor$1$1.onClose(CensusStatsModule.java:802)
> at
> io.grpc.PartialForwardingClientCallListener.onClose(PartialForwardingClientCallListener.java:39)
> at
> io.grpc.ForwardingClientCallListener.onClose(ForwardingClientCallListener.java:23)
> at
> io.grpc.ForwardingClientCallListener$SimpleForwardingClientCallListener.onClose(ForwardingClientCallListener.java:40)
> at
> io.grpc.census.CensusTracingModule$TracingClientInterceptor$1$1.onClose(CensusTracingModule.java:428)
> at io.grpc.internal.ClientCallImpl.closeObserver(ClientCallImpl.java:562)
> at io.grpc.internal.ClientCallImpl.access$300(ClientCallImpl.java:70) at
> io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInternal(ClientCallImpl.java:743)
> at
> io.grpc.internal.ClientCallImpl$ClientStreamListenerImpl$1StreamClosed.runInContext(ClientCallImpl.java:722)
> at io.grpc.internal.ContextRunnable.run(ContextRunnable.java:37) at
> io.grpc.internal.SerializingExecutor.run(SerializingExecutor.java:133) at
> java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at
> java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:829)
>