You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hudi.apache.org by Y Ethan Guo <et...@gmail.com> on 2020/07/26 23:12:23 UTC
Unit tests in hudi-client module fail due to SparkContext
Hi,
I'm working on hudi-client module and I notice that if I run all unit tests
under hudi-client locally in IntelliJ, some tests (54 out of 256) are
failing due to the following SparkException: "Only one SparkContext may be
running in this JVM". Is there any way I can get around this?
org.apache.spark.SparkException: Only one SparkContext may be running in
this JVM (see SPARK-2243). To ignore this error, set
spark.driver.allowMultipleContexts = true. The currently running
SparkContext was created at:
org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
java.lang.reflect.Method.invoke(Method.java:498)
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at
org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
at
org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
at scala.Option.foreach(Option.scala:257)
at
org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
at
org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
at
org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
at
org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
at
org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at
org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
at
org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
at
org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
at
org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
at
org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
at
org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
at
org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
at
org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
at
org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
at
org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
at java.util.ArrayList.forEach(ArrayList.java:1249)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
at java.util.ArrayList.forEach(ArrayList.java:1249)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
at
org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
at
org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
at
org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
at
org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
at
org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
at
org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
at
com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
at
com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
at
com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
Best,
- Ethan
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Vinoth Chandar <vi...@apache.org>.
Thanks for being so awesome, Raymond!
On Tue, Jul 28, 2020 at 4:23 PM Shiyan Xu <xu...@gmail.com>
wrote:
> yup i can make a PR for this.
>
> On Tue, Jul 28, 2020 at 2:30 PM Vinoth Chandar <vi...@apache.org> wrote:
>
> > Makes sense. Can we update some docs with this IDE setup?
> >
> > On Tue, Jul 28, 2020 at 10:32 AM Shiyan Xu <xu...@gmail.com>
> > wrote:
> >
> > > Sure... here it is
> > > https://gist.github.com/xushiyan/db4d4067657abe6b8872ef12473b7087
> > >
> > > On Tue, Jul 28, 2020 at 9:53 AM Vinoth Chandar <vi...@apache.org>
> > wrote:
> > >
> > > > Unfortunately, mailing list does not support images . you could
> create
> > a
> > > > gist and paste link :)
> > > >
> > > > On Tue, Jul 28, 2020 at 9:49 AM Y Ethan Guo <
> ethan.guoyihua@gmail.com>
> > > > wrote:
> > > >
> > > > > Thanks, Shiyan. Interesting, maybe the memory config is the issue
> > for
> > > > me.
> > > > > When I tried to run all tests, it fails. But running a smaller set
> > of
> > > > > tests per run is fine for me. I'll check that config in my
> settings.
> > > > >
> > > > > The discrepancy between the intellij test runner and CLI mvn runner
> > may
> > > > be
> > > > > > affected via these settings
> > > > >
> > > > > Somehow I can't see the screenshots...
> > > > >
> > > > > Thanks,
> > > > > - Ethan
> > > > >
> > > > > On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <
> > xu.shiyan.raymond@gmail.com
> > > >
> > > > > wrote:
> > > > >
> > > > > > At least that's what master build is setting with...
> > > > > >
> > > > > > The discrepancy between the intellij test runner and CLI mvn
> runner
> > > may
> > > > > be
> > > > > > affected via these settings
> > > > > > [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> > > > > > [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
> > > > > >
> > > > > > On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <
> > > xu.shiyan.raymond@gmail.com
> > > > >
> > > > > > wrote:
> > > > > >
> > > > > >> The maven surefire/failsafe plugin is configured with -Xmx2g
> here
> > > > > >> <
> > > > >
> > > >
> > >
> >
> https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124
> > > > > >,
> > > > > >> which should be plentiful for all tests so far.
> > > > > >> The OOM looks weird to me.. maybe try checking the maven log see
> > if
> > > > > >> -Xmx2g is indeed applied
> > > > > >>
> > > > > >> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <
> > > > ethan.guoyihua@gmail.com>
> > > > > >> wrote:
> > > > > >>
> > > > > >>> I see. I'll check the travis CI setup later. I'm unblocked
> now
> > > for
> > > > > >>> running the unit tests locally.
> > > > > >>>
> > > > > >>> Thanks,
> > > > > >>> - Ethan
> > > > > >>>
> > > > > >>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <
> > vinoth@apache.org
> > > >
> > > > > >>> wrote:
> > > > > >>>
> > > > > >>> > So, i realized something. With the recent changes, functional
> > > tests
> > > > > >>> retain
> > > > > >>> > a single spark session for the entire test suite to speed
> > things
> > > > up.
> > > > > >>> So
> > > > > >>> > thats probably what you were hitting first, when running via
> > IDE
> > > > > >>> > Try following the .travis.yml profiles directly?
> > > > > >>> >
> > > > > >>> >
> > > > > >>> > Not sure about the OOM. Have not seen this before.
> > > > > >>> >
> > > > > >>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <
> > > > > ethan.guoyihua@gmail.com
> > > > > >>> >
> > > > > >>> > wrote:
> > > > > >>> >
> > > > > >>> > > Thanks for the suggestion, Vinoth.
> > > > > >>> > >
> > > > > >>> > > I tried a few things on master:
> > > > > >>> > > - "mvn clean package -DskipITs": It throws the following
> > > > exception:
> > > > > >>> > >
> > > > > >>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*,
> > Skipped:
> > > 0,
> > > > > >>> Time
> > > > > >>> > > elapsed: 19.286 s* <<< FAILURE!* - in
> > > > > >>> > > org.apache.hudi.table.action.rollback.
> > > > > >>> > > *TestMergeOnReadRollbackActionExecutor*
> > > > > >>> > >
> > > > > >>> > > [*ERROR*]
> > > > > >>> > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > > > > >>> > > Time elapsed: 4.977 s <<< ERROR!
> > > > > >>> > >
> > > > > >>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed
> > to
> > > > > >>> respond
> > > > > >>> > >
> > > > > >>> > > at
> > > > > >>> > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > > > >>> > >
> > > > > >>> > > Caused by: org.apache.http.NoHttpResponseException: ...
> > failed
> > > to
> > > > > >>> respond
> > > > > >>> > >
> > > > > >>> > > at
> > > > > >>> > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > > > >>> > >
> > > > > >>> > > [*ERROR*]
> > > > > >>> > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > > > > >>> > > Time elapsed: 1.182 s <<< ERROR!
> > > > > >>> > >
> > > > > >>> > > org.apache.spark.SparkException:
> > > > > >>> > >
> > > > > >>> > > Only one SparkContext may be running in this JVM (see
> > > > SPARK-2243).
> > > > > To
> > > > > >>> > > ignore this error, set spark.driver.allowMultipleContexts =
> > > true.
> > > > > The
> > > > > >>> > > currently running SparkContext was created at:
> > > > > >>> > >
> > > > > >>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It
> > > > encounters
> > > > > >>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> > > > > >>> > >
> > > > > >>> > > What I do now is running tests under different packages in
> > > > > >>> hudi-client
> > > > > >>> > > manually in IntelliJ. That seems to work for me.
> > > > > >>> > >
> > > > > >>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <
> > > > vinoth@apache.org>
> > > > > >>> > wrote:
> > > > > >>> > >
> > > > > >>> > > > Hi Ethan,
> > > > > >>> > > >
> > > > > >>> > > > For purposes of unblocking yourself, can you try running
> > them
> > > > > >>> locally
> > > > > >>> > via
> > > > > >>> > > > mvn command via terminal?
> > > > > >>> > > >
> > > > > >>> > > > thanks
> > > > > >>> > > > vinoth
> > > > > >>> > > >
> > > > > >>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> > > > > >>> ethan.guoyihua@gmail.com>
> > > > > >>> > > > wrote:
> > > > > >>> > > >
> > > > > >>> > > > > Hi,
> > > > > >>> > > > >
> > > > > >>> > > > > I'm working on hudi-client module and I notice that if
> I
> > > run
> > > > > all
> > > > > >>> unit
> > > > > >>> > > > tests
> > > > > >>> > > > > under hudi-client locally in IntelliJ, some tests (54
> out
> > > of
> > > > > >>> 256) are
> > > > > >>> > > > > failing due to the following SparkException: "Only one
> > > > > >>> SparkContext
> > > > > >>> > may
> > > > > >>> > > > be
> > > > > >>> > > > > running in this JVM". Is there any way I can get
> around
> > > > this?
> > > > > >>> > > > >
> > > > > >>> > > > > org.apache.spark.SparkException: Only one SparkContext
> > may
> > > be
> > > > > >>> running
> > > > > >>> > > in
> > > > > >>> > > > > this JVM (see SPARK-2243). To ignore this error, set
> > > > > >>> > > > > spark.driver.allowMultipleContexts = true. The
> currently
> > > > > running
> > > > > >>> > > > > SparkContext was created at:
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > > > >>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > > >>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > > >>> > > > >
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > > > >>> > > > > at scala.Option.foreach(Option.scala:257)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > > > >>> > > > > at
> > > > org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > > > >>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > > Method)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > > >>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > >
> > > > > >>>
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > >
> > > > > >>>
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > >
> > > > > >>>
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > > > >>> > > > > at
> > > > > >>> > > > >
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > >
> > > >
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > > > >>> > > > > at
> > > > > com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > > > > >>> > > > >
> > > > > >>> > > > > Best,
> > > > > >>> > > > > - Ethan
> > > > > >>> > > > >
> > > > > >>> > > >
> > > > > >>> > >
> > > > > >>> >
> > > > > >>>
> > > > > >>
> > > > >
> > > >
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Shiyan Xu <xu...@gmail.com>.
yup i can make a PR for this.
On Tue, Jul 28, 2020 at 2:30 PM Vinoth Chandar <vi...@apache.org> wrote:
> Makes sense. Can we update some docs with this IDE setup?
>
> On Tue, Jul 28, 2020 at 10:32 AM Shiyan Xu <xu...@gmail.com>
> wrote:
>
> > Sure... here it is
> > https://gist.github.com/xushiyan/db4d4067657abe6b8872ef12473b7087
> >
> > On Tue, Jul 28, 2020 at 9:53 AM Vinoth Chandar <vi...@apache.org>
> wrote:
> >
> > > Unfortunately, mailing list does not support images . you could create
> a
> > > gist and paste link :)
> > >
> > > On Tue, Jul 28, 2020 at 9:49 AM Y Ethan Guo <et...@gmail.com>
> > > wrote:
> > >
> > > > Thanks, Shiyan. Interesting, maybe the memory config is the issue
> for
> > > me.
> > > > When I tried to run all tests, it fails. But running a smaller set
> of
> > > > tests per run is fine for me. I'll check that config in my settings.
> > > >
> > > > The discrepancy between the intellij test runner and CLI mvn runner
> may
> > > be
> > > > > affected via these settings
> > > >
> > > > Somehow I can't see the screenshots...
> > > >
> > > > Thanks,
> > > > - Ethan
> > > >
> > > > On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <
> xu.shiyan.raymond@gmail.com
> > >
> > > > wrote:
> > > >
> > > > > At least that's what master build is setting with...
> > > > >
> > > > > The discrepancy between the intellij test runner and CLI mvn runner
> > may
> > > > be
> > > > > affected via these settings
> > > > > [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> > > > > [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
> > > > >
> > > > > On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <
> > xu.shiyan.raymond@gmail.com
> > > >
> > > > > wrote:
> > > > >
> > > > >> The maven surefire/failsafe plugin is configured with -Xmx2g here
> > > > >> <
> > > >
> > >
> >
> https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124
> > > > >,
> > > > >> which should be plentiful for all tests so far.
> > > > >> The OOM looks weird to me.. maybe try checking the maven log see
> if
> > > > >> -Xmx2g is indeed applied
> > > > >>
> > > > >> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <
> > > ethan.guoyihua@gmail.com>
> > > > >> wrote:
> > > > >>
> > > > >>> I see. I'll check the travis CI setup later. I'm unblocked now
> > for
> > > > >>> running the unit tests locally.
> > > > >>>
> > > > >>> Thanks,
> > > > >>> - Ethan
> > > > >>>
> > > > >>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <
> vinoth@apache.org
> > >
> > > > >>> wrote:
> > > > >>>
> > > > >>> > So, i realized something. With the recent changes, functional
> > tests
> > > > >>> retain
> > > > >>> > a single spark session for the entire test suite to speed
> things
> > > up.
> > > > >>> So
> > > > >>> > thats probably what you were hitting first, when running via
> IDE
> > > > >>> > Try following the .travis.yml profiles directly?
> > > > >>> >
> > > > >>> >
> > > > >>> > Not sure about the OOM. Have not seen this before.
> > > > >>> >
> > > > >>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <
> > > > ethan.guoyihua@gmail.com
> > > > >>> >
> > > > >>> > wrote:
> > > > >>> >
> > > > >>> > > Thanks for the suggestion, Vinoth.
> > > > >>> > >
> > > > >>> > > I tried a few things on master:
> > > > >>> > > - "mvn clean package -DskipITs": It throws the following
> > > exception:
> > > > >>> > >
> > > > >>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*,
> Skipped:
> > 0,
> > > > >>> Time
> > > > >>> > > elapsed: 19.286 s* <<< FAILURE!* - in
> > > > >>> > > org.apache.hudi.table.action.rollback.
> > > > >>> > > *TestMergeOnReadRollbackActionExecutor*
> > > > >>> > >
> > > > >>> > > [*ERROR*]
> > > > >>> > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > > > >>> > > Time elapsed: 4.977 s <<< ERROR!
> > > > >>> > >
> > > > >>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed
> to
> > > > >>> respond
> > > > >>> > >
> > > > >>> > > at
> > > > >>> > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > > >>> > >
> > > > >>> > > Caused by: org.apache.http.NoHttpResponseException: ...
> failed
> > to
> > > > >>> respond
> > > > >>> > >
> > > > >>> > > at
> > > > >>> > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > > >>> > >
> > > > >>> > > [*ERROR*]
> > > > >>> > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > > > >>> > > Time elapsed: 1.182 s <<< ERROR!
> > > > >>> > >
> > > > >>> > > org.apache.spark.SparkException:
> > > > >>> > >
> > > > >>> > > Only one SparkContext may be running in this JVM (see
> > > SPARK-2243).
> > > > To
> > > > >>> > > ignore this error, set spark.driver.allowMultipleContexts =
> > true.
> > > > The
> > > > >>> > > currently running SparkContext was created at:
> > > > >>> > >
> > > > >>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It
> > > encounters
> > > > >>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> > > > >>> > >
> > > > >>> > > What I do now is running tests under different packages in
> > > > >>> hudi-client
> > > > >>> > > manually in IntelliJ. That seems to work for me.
> > > > >>> > >
> > > > >>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <
> > > vinoth@apache.org>
> > > > >>> > wrote:
> > > > >>> > >
> > > > >>> > > > Hi Ethan,
> > > > >>> > > >
> > > > >>> > > > For purposes of unblocking yourself, can you try running
> them
> > > > >>> locally
> > > > >>> > via
> > > > >>> > > > mvn command via terminal?
> > > > >>> > > >
> > > > >>> > > > thanks
> > > > >>> > > > vinoth
> > > > >>> > > >
> > > > >>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> > > > >>> ethan.guoyihua@gmail.com>
> > > > >>> > > > wrote:
> > > > >>> > > >
> > > > >>> > > > > Hi,
> > > > >>> > > > >
> > > > >>> > > > > I'm working on hudi-client module and I notice that if I
> > run
> > > > all
> > > > >>> unit
> > > > >>> > > > tests
> > > > >>> > > > > under hudi-client locally in IntelliJ, some tests (54 out
> > of
> > > > >>> 256) are
> > > > >>> > > > > failing due to the following SparkException: "Only one
> > > > >>> SparkContext
> > > > >>> > may
> > > > >>> > > > be
> > > > >>> > > > > running in this JVM". Is there any way I can get around
> > > this?
> > > > >>> > > > >
> > > > >>> > > > > org.apache.spark.SparkException: Only one SparkContext
> may
> > be
> > > > >>> running
> > > > >>> > > in
> > > > >>> > > > > this JVM (see SPARK-2243). To ignore this error, set
> > > > >>> > > > > spark.driver.allowMultipleContexts = true. The currently
> > > > running
> > > > >>> > > > > SparkContext was created at:
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > > >>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > >>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > >>> > > > >
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > > >>> > > > > at scala.Option.foreach(Option.scala:257)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > > >>> > > > > at
> > > org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > > >>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > > Method)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > >>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > >
> > > > >>>
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > >
> > > > >>>
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > >
> > > > >>>
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > > >>> > > > > at
> > > > >>> > > > >
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > >
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > > >>> > > > > at
> > > > com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > > > >>> > > > >
> > > > >>> > > > > Best,
> > > > >>> > > > > - Ethan
> > > > >>> > > > >
> > > > >>> > > >
> > > > >>> > >
> > > > >>> >
> > > > >>>
> > > > >>
> > > >
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Vinoth Chandar <vi...@apache.org>.
Makes sense. Can we update some docs with this IDE setup?
On Tue, Jul 28, 2020 at 10:32 AM Shiyan Xu <xu...@gmail.com>
wrote:
> Sure... here it is
> https://gist.github.com/xushiyan/db4d4067657abe6b8872ef12473b7087
>
> On Tue, Jul 28, 2020 at 9:53 AM Vinoth Chandar <vi...@apache.org> wrote:
>
> > Unfortunately, mailing list does not support images . you could create a
> > gist and paste link :)
> >
> > On Tue, Jul 28, 2020 at 9:49 AM Y Ethan Guo <et...@gmail.com>
> > wrote:
> >
> > > Thanks, Shiyan. Interesting, maybe the memory config is the issue for
> > me.
> > > When I tried to run all tests, it fails. But running a smaller set of
> > > tests per run is fine for me. I'll check that config in my settings.
> > >
> > > The discrepancy between the intellij test runner and CLI mvn runner may
> > be
> > > > affected via these settings
> > >
> > > Somehow I can't see the screenshots...
> > >
> > > Thanks,
> > > - Ethan
> > >
> > > On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <xu.shiyan.raymond@gmail.com
> >
> > > wrote:
> > >
> > > > At least that's what master build is setting with...
> > > >
> > > > The discrepancy between the intellij test runner and CLI mvn runner
> may
> > > be
> > > > affected via these settings
> > > > [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> > > > [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
> > > >
> > > > On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <
> xu.shiyan.raymond@gmail.com
> > >
> > > > wrote:
> > > >
> > > >> The maven surefire/failsafe plugin is configured with -Xmx2g here
> > > >> <
> > >
> >
> https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124
> > > >,
> > > >> which should be plentiful for all tests so far.
> > > >> The OOM looks weird to me.. maybe try checking the maven log see if
> > > >> -Xmx2g is indeed applied
> > > >>
> > > >> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <
> > ethan.guoyihua@gmail.com>
> > > >> wrote:
> > > >>
> > > >>> I see. I'll check the travis CI setup later. I'm unblocked now
> for
> > > >>> running the unit tests locally.
> > > >>>
> > > >>> Thanks,
> > > >>> - Ethan
> > > >>>
> > > >>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vinoth@apache.org
> >
> > > >>> wrote:
> > > >>>
> > > >>> > So, i realized something. With the recent changes, functional
> tests
> > > >>> retain
> > > >>> > a single spark session for the entire test suite to speed things
> > up.
> > > >>> So
> > > >>> > thats probably what you were hitting first, when running via IDE
> > > >>> > Try following the .travis.yml profiles directly?
> > > >>> >
> > > >>> >
> > > >>> > Not sure about the OOM. Have not seen this before.
> > > >>> >
> > > >>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <
> > > ethan.guoyihua@gmail.com
> > > >>> >
> > > >>> > wrote:
> > > >>> >
> > > >>> > > Thanks for the suggestion, Vinoth.
> > > >>> > >
> > > >>> > > I tried a few things on master:
> > > >>> > > - "mvn clean package -DskipITs": It throws the following
> > exception:
> > > >>> > >
> > > >>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped:
> 0,
> > > >>> Time
> > > >>> > > elapsed: 19.286 s* <<< FAILURE!* - in
> > > >>> > > org.apache.hudi.table.action.rollback.
> > > >>> > > *TestMergeOnReadRollbackActionExecutor*
> > > >>> > >
> > > >>> > > [*ERROR*]
> > > >>> > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > > >>> > > Time elapsed: 4.977 s <<< ERROR!
> > > >>> > >
> > > >>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to
> > > >>> respond
> > > >>> > >
> > > >>> > > at
> > > >>> > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > >>> > >
> > > >>> > > Caused by: org.apache.http.NoHttpResponseException: ... failed
> to
> > > >>> respond
> > > >>> > >
> > > >>> > > at
> > > >>> > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > > >>> > >
> > > >>> > > [*ERROR*]
> > > >>> > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > > >>> > > Time elapsed: 1.182 s <<< ERROR!
> > > >>> > >
> > > >>> > > org.apache.spark.SparkException:
> > > >>> > >
> > > >>> > > Only one SparkContext may be running in this JVM (see
> > SPARK-2243).
> > > To
> > > >>> > > ignore this error, set spark.driver.allowMultipleContexts =
> true.
> > > The
> > > >>> > > currently running SparkContext was created at:
> > > >>> > >
> > > >>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It
> > encounters
> > > >>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> > > >>> > >
> > > >>> > > What I do now is running tests under different packages in
> > > >>> hudi-client
> > > >>> > > manually in IntelliJ. That seems to work for me.
> > > >>> > >
> > > >>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <
> > vinoth@apache.org>
> > > >>> > wrote:
> > > >>> > >
> > > >>> > > > Hi Ethan,
> > > >>> > > >
> > > >>> > > > For purposes of unblocking yourself, can you try running them
> > > >>> locally
> > > >>> > via
> > > >>> > > > mvn command via terminal?
> > > >>> > > >
> > > >>> > > > thanks
> > > >>> > > > vinoth
> > > >>> > > >
> > > >>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> > > >>> ethan.guoyihua@gmail.com>
> > > >>> > > > wrote:
> > > >>> > > >
> > > >>> > > > > Hi,
> > > >>> > > > >
> > > >>> > > > > I'm working on hudi-client module and I notice that if I
> run
> > > all
> > > >>> unit
> > > >>> > > > tests
> > > >>> > > > > under hudi-client locally in IntelliJ, some tests (54 out
> of
> > > >>> 256) are
> > > >>> > > > > failing due to the following SparkException: "Only one
> > > >>> SparkContext
> > > >>> > may
> > > >>> > > > be
> > > >>> > > > > running in this JVM". Is there any way I can get around
> > this?
> > > >>> > > > >
> > > >>> > > > > org.apache.spark.SparkException: Only one SparkContext may
> be
> > > >>> running
> > > >>> > > in
> > > >>> > > > > this JVM (see SPARK-2243). To ignore this error, set
> > > >>> > > > > spark.driver.allowMultipleContexts = true. The currently
> > > running
> > > >>> > > > > SparkContext was created at:
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > >>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > >>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > >>> > > > >
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > >>> > > > > at scala.Option.foreach(Option.scala:257)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > >>> > > > > at
> > org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > >>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> > Method)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > >>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > >
> > > >>>
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > >
> > > >>>
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > >
> > > >>>
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > >>> > > > > at
> > > >>> > > > >
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > >>> > > > > at
> > > com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > > >>> > > > >
> > > >>> > > > > Best,
> > > >>> > > > > - Ethan
> > > >>> > > > >
> > > >>> > > >
> > > >>> > >
> > > >>> >
> > > >>>
> > > >>
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Shiyan Xu <xu...@gmail.com>.
Sure... here it is
https://gist.github.com/xushiyan/db4d4067657abe6b8872ef12473b7087
On Tue, Jul 28, 2020 at 9:53 AM Vinoth Chandar <vi...@apache.org> wrote:
> Unfortunately, mailing list does not support images . you could create a
> gist and paste link :)
>
> On Tue, Jul 28, 2020 at 9:49 AM Y Ethan Guo <et...@gmail.com>
> wrote:
>
> > Thanks, Shiyan. Interesting, maybe the memory config is the issue for
> me.
> > When I tried to run all tests, it fails. But running a smaller set of
> > tests per run is fine for me. I'll check that config in my settings.
> >
> > The discrepancy between the intellij test runner and CLI mvn runner may
> be
> > > affected via these settings
> >
> > Somehow I can't see the screenshots...
> >
> > Thanks,
> > - Ethan
> >
> > On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <xu...@gmail.com>
> > wrote:
> >
> > > At least that's what master build is setting with...
> > >
> > > The discrepancy between the intellij test runner and CLI mvn runner may
> > be
> > > affected via these settings
> > > [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> > > [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
> > >
> > > On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <xu.shiyan.raymond@gmail.com
> >
> > > wrote:
> > >
> > >> The maven surefire/failsafe plugin is configured with -Xmx2g here
> > >> <
> >
> https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124
> > >,
> > >> which should be plentiful for all tests so far.
> > >> The OOM looks weird to me.. maybe try checking the maven log see if
> > >> -Xmx2g is indeed applied
> > >>
> > >> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <
> ethan.guoyihua@gmail.com>
> > >> wrote:
> > >>
> > >>> I see. I'll check the travis CI setup later. I'm unblocked now for
> > >>> running the unit tests locally.
> > >>>
> > >>> Thanks,
> > >>> - Ethan
> > >>>
> > >>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org>
> > >>> wrote:
> > >>>
> > >>> > So, i realized something. With the recent changes, functional tests
> > >>> retain
> > >>> > a single spark session for the entire test suite to speed things
> up.
> > >>> So
> > >>> > thats probably what you were hitting first, when running via IDE
> > >>> > Try following the .travis.yml profiles directly?
> > >>> >
> > >>> >
> > >>> > Not sure about the OOM. Have not seen this before.
> > >>> >
> > >>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <
> > ethan.guoyihua@gmail.com
> > >>> >
> > >>> > wrote:
> > >>> >
> > >>> > > Thanks for the suggestion, Vinoth.
> > >>> > >
> > >>> > > I tried a few things on master:
> > >>> > > - "mvn clean package -DskipITs": It throws the following
> exception:
> > >>> > >
> > >>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0,
> > >>> Time
> > >>> > > elapsed: 19.286 s* <<< FAILURE!* - in
> > >>> > > org.apache.hudi.table.action.rollback.
> > >>> > > *TestMergeOnReadRollbackActionExecutor*
> > >>> > >
> > >>> > > [*ERROR*]
> > >>> > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > >>> > > Time elapsed: 4.977 s <<< ERROR!
> > >>> > >
> > >>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to
> > >>> respond
> > >>> > >
> > >>> > > at
> > >>> > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > >>> > >
> > >>> > > Caused by: org.apache.http.NoHttpResponseException: ... failed to
> > >>> respond
> > >>> > >
> > >>> > > at
> > >>> > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > >>> > >
> > >>> > > [*ERROR*]
> > >>> > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > >>> > > Time elapsed: 1.182 s <<< ERROR!
> > >>> > >
> > >>> > > org.apache.spark.SparkException:
> > >>> > >
> > >>> > > Only one SparkContext may be running in this JVM (see
> SPARK-2243).
> > To
> > >>> > > ignore this error, set spark.driver.allowMultipleContexts = true.
> > The
> > >>> > > currently running SparkContext was created at:
> > >>> > >
> > >>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It
> encounters
> > >>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> > >>> > >
> > >>> > > What I do now is running tests under different packages in
> > >>> hudi-client
> > >>> > > manually in IntelliJ. That seems to work for me.
> > >>> > >
> > >>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <
> vinoth@apache.org>
> > >>> > wrote:
> > >>> > >
> > >>> > > > Hi Ethan,
> > >>> > > >
> > >>> > > > For purposes of unblocking yourself, can you try running them
> > >>> locally
> > >>> > via
> > >>> > > > mvn command via terminal?
> > >>> > > >
> > >>> > > > thanks
> > >>> > > > vinoth
> > >>> > > >
> > >>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> > >>> ethan.guoyihua@gmail.com>
> > >>> > > > wrote:
> > >>> > > >
> > >>> > > > > Hi,
> > >>> > > > >
> > >>> > > > > I'm working on hudi-client module and I notice that if I run
> > all
> > >>> unit
> > >>> > > > tests
> > >>> > > > > under hudi-client locally in IntelliJ, some tests (54 out of
> > >>> 256) are
> > >>> > > > > failing due to the following SparkException: "Only one
> > >>> SparkContext
> > >>> > may
> > >>> > > > be
> > >>> > > > > running in this JVM". Is there any way I can get around
> this?
> > >>> > > > >
> > >>> > > > > org.apache.spark.SparkException: Only one SparkContext may be
> > >>> running
> > >>> > > in
> > >>> > > > > this JVM (see SPARK-2243). To ignore this error, set
> > >>> > > > > spark.driver.allowMultipleContexts = true. The currently
> > running
> > >>> > > > > SparkContext was created at:
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > >>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > >>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > >>> > > > >
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > >>> > > > > at scala.Option.foreach(Option.scala:257)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > >>> > > > > at
> org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > >>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native
> Method)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > >>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > >>> > > > > at
> > >>> > > > >
> > >>> > >
> > >>>
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > >>> > > > > at
> > >>> > > > >
> > >>> > >
> > >>>
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > >>> > > > > at
> > >>> > > > >
> > >>> > >
> > >>>
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > >>> > > > > at
> > >>> > > > >
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > >>> > > > > at
> > com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > >>> > > > >
> > >>> > > > > Best,
> > >>> > > > > - Ethan
> > >>> > > > >
> > >>> > > >
> > >>> > >
> > >>> >
> > >>>
> > >>
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Vinoth Chandar <vi...@apache.org>.
Unfortunately, mailing list does not support images . you could create a
gist and paste link :)
On Tue, Jul 28, 2020 at 9:49 AM Y Ethan Guo <et...@gmail.com>
wrote:
> Thanks, Shiyan. Interesting, maybe the memory config is the issue for me.
> When I tried to run all tests, it fails. But running a smaller set of
> tests per run is fine for me. I'll check that config in my settings.
>
> The discrepancy between the intellij test runner and CLI mvn runner may be
> > affected via these settings
>
> Somehow I can't see the screenshots...
>
> Thanks,
> - Ethan
>
> On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <xu...@gmail.com>
> wrote:
>
> > At least that's what master build is setting with...
> >
> > The discrepancy between the intellij test runner and CLI mvn runner may
> be
> > affected via these settings
> > [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> > [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
> >
> > On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <xu...@gmail.com>
> > wrote:
> >
> >> The maven surefire/failsafe plugin is configured with -Xmx2g here
> >> <
> https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124
> >,
> >> which should be plentiful for all tests so far.
> >> The OOM looks weird to me.. maybe try checking the maven log see if
> >> -Xmx2g is indeed applied
> >>
> >> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <et...@gmail.com>
> >> wrote:
> >>
> >>> I see. I'll check the travis CI setup later. I'm unblocked now for
> >>> running the unit tests locally.
> >>>
> >>> Thanks,
> >>> - Ethan
> >>>
> >>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org>
> >>> wrote:
> >>>
> >>> > So, i realized something. With the recent changes, functional tests
> >>> retain
> >>> > a single spark session for the entire test suite to speed things up.
> >>> So
> >>> > thats probably what you were hitting first, when running via IDE
> >>> > Try following the .travis.yml profiles directly?
> >>> >
> >>> >
> >>> > Not sure about the OOM. Have not seen this before.
> >>> >
> >>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <
> ethan.guoyihua@gmail.com
> >>> >
> >>> > wrote:
> >>> >
> >>> > > Thanks for the suggestion, Vinoth.
> >>> > >
> >>> > > I tried a few things on master:
> >>> > > - "mvn clean package -DskipITs": It throws the following exception:
> >>> > >
> >>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0,
> >>> Time
> >>> > > elapsed: 19.286 s* <<< FAILURE!* - in
> >>> > > org.apache.hudi.table.action.rollback.
> >>> > > *TestMergeOnReadRollbackActionExecutor*
> >>> > >
> >>> > > [*ERROR*]
> >>> > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> >>> > > Time elapsed: 4.977 s <<< ERROR!
> >>> > >
> >>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to
> >>> respond
> >>> > >
> >>> > > at
> >>> > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> >>> > >
> >>> > > Caused by: org.apache.http.NoHttpResponseException: ... failed to
> >>> respond
> >>> > >
> >>> > > at
> >>> > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> >>> > >
> >>> > > [*ERROR*]
> >>> > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> >>> > > Time elapsed: 1.182 s <<< ERROR!
> >>> > >
> >>> > > org.apache.spark.SparkException:
> >>> > >
> >>> > > Only one SparkContext may be running in this JVM (see SPARK-2243).
> To
> >>> > > ignore this error, set spark.driver.allowMultipleContexts = true.
> The
> >>> > > currently running SparkContext was created at:
> >>> > >
> >>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
> >>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> >>> > >
> >>> > > What I do now is running tests under different packages in
> >>> hudi-client
> >>> > > manually in IntelliJ. That seems to work for me.
> >>> > >
> >>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org>
> >>> > wrote:
> >>> > >
> >>> > > > Hi Ethan,
> >>> > > >
> >>> > > > For purposes of unblocking yourself, can you try running them
> >>> locally
> >>> > via
> >>> > > > mvn command via terminal?
> >>> > > >
> >>> > > > thanks
> >>> > > > vinoth
> >>> > > >
> >>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> >>> ethan.guoyihua@gmail.com>
> >>> > > > wrote:
> >>> > > >
> >>> > > > > Hi,
> >>> > > > >
> >>> > > > > I'm working on hudi-client module and I notice that if I run
> all
> >>> unit
> >>> > > > tests
> >>> > > > > under hudi-client locally in IntelliJ, some tests (54 out of
> >>> 256) are
> >>> > > > > failing due to the following SparkException: "Only one
> >>> SparkContext
> >>> > may
> >>> > > > be
> >>> > > > > running in this JVM". Is there any way I can get around this?
> >>> > > > >
> >>> > > > > org.apache.spark.SparkException: Only one SparkContext may be
> >>> running
> >>> > > in
> >>> > > > > this JVM (see SPARK-2243). To ignore this error, set
> >>> > > > > spark.driver.allowMultipleContexts = true. The currently
> running
> >>> > > > > SparkContext was created at:
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> >>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> >>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> >>> > > > >
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> >>> > > > > at scala.Option.foreach(Option.scala:257)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> >>> > > > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> >>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> >>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> >>> > > > > at
> >>> > > > >
> >>> > >
> >>>
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> >>> > > > > at
> >>> > > > >
> >>> > >
> >>>
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> >>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> >>> > > > > at
> >>> > > > >
> >>> > >
> >>>
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> >>> > > > > at
> >>> > > > >
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> >>> > > > > at
> com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> >>> > > > >
> >>> > > > > Best,
> >>> > > > > - Ethan
> >>> > > > >
> >>> > > >
> >>> > >
> >>> >
> >>>
> >>
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Y Ethan Guo <et...@gmail.com>.
Thanks, Shiyan. Interesting, maybe the memory config is the issue for me.
When I tried to run all tests, it fails. But running a smaller set of
tests per run is fine for me. I'll check that config in my settings.
The discrepancy between the intellij test runner and CLI mvn runner may be
> affected via these settings
Somehow I can't see the screenshots...
Thanks,
- Ethan
On Tue, Jul 28, 2020 at 9:32 AM Shiyan Xu <xu...@gmail.com>
wrote:
> At least that's what master build is setting with...
>
> The discrepancy between the intellij test runner and CLI mvn runner may be
> affected via these settings
> [image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
> [image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
>
> On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <xu...@gmail.com>
> wrote:
>
>> The maven surefire/failsafe plugin is configured with -Xmx2g here
>> <https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124>,
>> which should be plentiful for all tests so far.
>> The OOM looks weird to me.. maybe try checking the maven log see if
>> -Xmx2g is indeed applied
>>
>> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <et...@gmail.com>
>> wrote:
>>
>>> I see. I'll check the travis CI setup later. I'm unblocked now for
>>> running the unit tests locally.
>>>
>>> Thanks,
>>> - Ethan
>>>
>>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org>
>>> wrote:
>>>
>>> > So, i realized something. With the recent changes, functional tests
>>> retain
>>> > a single spark session for the entire test suite to speed things up.
>>> So
>>> > thats probably what you were hitting first, when running via IDE
>>> > Try following the .travis.yml profiles directly?
>>> >
>>> >
>>> > Not sure about the OOM. Have not seen this before.
>>> >
>>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <ethan.guoyihua@gmail.com
>>> >
>>> > wrote:
>>> >
>>> > > Thanks for the suggestion, Vinoth.
>>> > >
>>> > > I tried a few things on master:
>>> > > - "mvn clean package -DskipITs": It throws the following exception:
>>> > >
>>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0,
>>> Time
>>> > > elapsed: 19.286 s* <<< FAILURE!* - in
>>> > > org.apache.hudi.table.action.rollback.
>>> > > *TestMergeOnReadRollbackActionExecutor*
>>> > >
>>> > > [*ERROR*]
>>> > >
>>> > >
>>> >
>>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
>>> > > Time elapsed: 4.977 s <<< ERROR!
>>> > >
>>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to
>>> respond
>>> > >
>>> > > at
>>> > >
>>> > >
>>> >
>>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>>> > >
>>> > > Caused by: org.apache.http.NoHttpResponseException: ... failed to
>>> respond
>>> > >
>>> > > at
>>> > >
>>> > >
>>> >
>>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>>> > >
>>> > > [*ERROR*]
>>> > >
>>> > >
>>> >
>>> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
>>> > > Time elapsed: 1.182 s <<< ERROR!
>>> > >
>>> > > org.apache.spark.SparkException:
>>> > >
>>> > > Only one SparkContext may be running in this JVM (see SPARK-2243). To
>>> > > ignore this error, set spark.driver.allowMultipleContexts = true. The
>>> > > currently running SparkContext was created at:
>>> > >
>>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
>>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
>>> > >
>>> > > What I do now is running tests under different packages in
>>> hudi-client
>>> > > manually in IntelliJ. That seems to work for me.
>>> > >
>>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org>
>>> > wrote:
>>> > >
>>> > > > Hi Ethan,
>>> > > >
>>> > > > For purposes of unblocking yourself, can you try running them
>>> locally
>>> > via
>>> > > > mvn command via terminal?
>>> > > >
>>> > > > thanks
>>> > > > vinoth
>>> > > >
>>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
>>> ethan.guoyihua@gmail.com>
>>> > > > wrote:
>>> > > >
>>> > > > > Hi,
>>> > > > >
>>> > > > > I'm working on hudi-client module and I notice that if I run all
>>> unit
>>> > > > tests
>>> > > > > under hudi-client locally in IntelliJ, some tests (54 out of
>>> 256) are
>>> > > > > failing due to the following SparkException: "Only one
>>> SparkContext
>>> > may
>>> > > > be
>>> > > > > running in this JVM". Is there any way I can get around this?
>>> > > > >
>>> > > > > org.apache.spark.SparkException: Only one SparkContext may be
>>> running
>>> > > in
>>> > > > > this JVM (see SPARK-2243). To ignore this error, set
>>> > > > > spark.driver.allowMultipleContexts = true. The currently running
>>> > > > > SparkContext was created at:
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
>>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
>>> > > > >
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
>>> > > > > at scala.Option.foreach(Option.scala:257)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
>>> > > > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
>>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>>> > > > > at
>>> > > > >
>>> > >
>>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>>> > > > > at
>>> > > > >
>>> > >
>>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>>> > > > > at
>>> > > > >
>>> > >
>>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
>>> > > > > at
>>> > > > >
>>> > > > >
>>> > > >
>>> > >
>>> >
>>> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
>>> > > > > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
>>> > > > >
>>> > > > > Best,
>>> > > > > - Ethan
>>> > > > >
>>> > > >
>>> > >
>>> >
>>>
>>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Shiyan Xu <xu...@gmail.com>.
At least that's what master build is setting with...
The discrepancy between the intellij test runner and CLI mvn runner may be
affected via these settings
[image: Screen Shot 2020-07-28 at 9.29.20 AM.png]
[image: Screen Shot 2020-07-28 at 9.29.27 AM.png]
On Tue, Jul 28, 2020 at 9:19 AM Shiyan Xu <xu...@gmail.com>
wrote:
> The maven surefire/failsafe plugin is configured with -Xmx2g here
> <https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124>,
> which should be plentiful for all tests so far.
> The OOM looks weird to me.. maybe try checking the maven log see if -Xmx2g
> is indeed applied
>
> On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <et...@gmail.com>
> wrote:
>
>> I see. I'll check the travis CI setup later. I'm unblocked now for
>> running the unit tests locally.
>>
>> Thanks,
>> - Ethan
>>
>> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org>
>> wrote:
>>
>> > So, i realized something. With the recent changes, functional tests
>> retain
>> > a single spark session for the entire test suite to speed things up. So
>> > thats probably what you were hitting first, when running via IDE
>> > Try following the .travis.yml profiles directly?
>> >
>> >
>> > Not sure about the OOM. Have not seen this before.
>> >
>> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <et...@gmail.com>
>> > wrote:
>> >
>> > > Thanks for the suggestion, Vinoth.
>> > >
>> > > I tried a few things on master:
>> > > - "mvn clean package -DskipITs": It throws the following exception:
>> > >
>> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0, Time
>> > > elapsed: 19.286 s* <<< FAILURE!* - in
>> > > org.apache.hudi.table.action.rollback.
>> > > *TestMergeOnReadRollbackActionExecutor*
>> > >
>> > > [*ERROR*]
>> > >
>> > >
>> >
>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
>> > > Time elapsed: 4.977 s <<< ERROR!
>> > >
>> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to respond
>> > >
>> > > at
>> > >
>> > >
>> >
>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>> > >
>> > > Caused by: org.apache.http.NoHttpResponseException: ... failed to
>> respond
>> > >
>> > > at
>> > >
>> > >
>> >
>> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>> > >
>> > > [*ERROR*]
>> > >
>> > >
>> >
>> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
>> > > Time elapsed: 1.182 s <<< ERROR!
>> > >
>> > > org.apache.spark.SparkException:
>> > >
>> > > Only one SparkContext may be running in this JVM (see SPARK-2243). To
>> > > ignore this error, set spark.driver.allowMultipleContexts = true. The
>> > > currently running SparkContext was created at:
>> > >
>> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
>> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
>> > >
>> > > What I do now is running tests under different packages in hudi-client
>> > > manually in IntelliJ. That seems to work for me.
>> > >
>> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org>
>> > wrote:
>> > >
>> > > > Hi Ethan,
>> > > >
>> > > > For purposes of unblocking yourself, can you try running them
>> locally
>> > via
>> > > > mvn command via terminal?
>> > > >
>> > > > thanks
>> > > > vinoth
>> > > >
>> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
>> ethan.guoyihua@gmail.com>
>> > > > wrote:
>> > > >
>> > > > > Hi,
>> > > > >
>> > > > > I'm working on hudi-client module and I notice that if I run all
>> unit
>> > > > tests
>> > > > > under hudi-client locally in IntelliJ, some tests (54 out of 256)
>> are
>> > > > > failing due to the following SparkException: "Only one
>> SparkContext
>> > may
>> > > > be
>> > > > > running in this JVM". Is there any way I can get around this?
>> > > > >
>> > > > > org.apache.spark.SparkException: Only one SparkContext may be
>> running
>> > > in
>> > > > > this JVM (see SPARK-2243). To ignore this error, set
>> > > > > spark.driver.allowMultipleContexts = true. The currently running
>> > > > > SparkContext was created at:
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
>> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> > > > > java.lang.reflect.Method.invoke(Method.java:498)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
>> > > > >
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
>> > > > > at scala.Option.foreach(Option.scala:257)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
>> > > > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
>> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
>> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>> > > > > at
>> > > > >
>> > >
>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>> > > > > at
>> > > > >
>> > >
>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
>> > > > > at
>> > > > >
>> > >
>> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
>> > > > > at
>> > > > >
>> > > > >
>> > > >
>> > >
>> >
>> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
>> > > > > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
>> > > > >
>> > > > > Best,
>> > > > > - Ethan
>> > > > >
>> > > >
>> > >
>> >
>>
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Shiyan Xu <xu...@gmail.com>.
The maven surefire/failsafe plugin is configured with -Xmx2g here
<https://github.com/apache/hudi/blob/b2763f433b3efb92fdcc0e760a88a43eaa2e5be3/pom.xml#L124>,
which should be plentiful for all tests so far.
The OOM looks weird to me.. maybe try checking the maven log see if -Xmx2g
is indeed applied
On Mon, Jul 27, 2020 at 11:11 PM Y Ethan Guo <et...@gmail.com>
wrote:
> I see. I'll check the travis CI setup later. I'm unblocked now for
> running the unit tests locally.
>
> Thanks,
> - Ethan
>
> On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org> wrote:
>
> > So, i realized something. With the recent changes, functional tests
> retain
> > a single spark session for the entire test suite to speed things up. So
> > thats probably what you were hitting first, when running via IDE
> > Try following the .travis.yml profiles directly?
> >
> >
> > Not sure about the OOM. Have not seen this before.
> >
> > On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <et...@gmail.com>
> > wrote:
> >
> > > Thanks for the suggestion, Vinoth.
> > >
> > > I tried a few things on master:
> > > - "mvn clean package -DskipITs": It throws the following exception:
> > >
> > > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0, Time
> > > elapsed: 19.286 s* <<< FAILURE!* - in
> > > org.apache.hudi.table.action.rollback.
> > > *TestMergeOnReadRollbackActionExecutor*
> > >
> > > [*ERROR*]
> > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > > Time elapsed: 4.977 s <<< ERROR!
> > >
> > > org.apache.hudi.exception.HoodieRemoteException: ... failed to respond
> > >
> > > at
> > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > >
> > > Caused by: org.apache.http.NoHttpResponseException: ... failed to
> respond
> > >
> > > at
> > >
> > >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> > >
> > > [*ERROR*]
> > >
> > >
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > > Time elapsed: 1.182 s <<< ERROR!
> > >
> > > org.apache.spark.SparkException:
> > >
> > > Only one SparkContext may be running in this JVM (see SPARK-2243). To
> > > ignore this error, set spark.driver.allowMultipleContexts = true. The
> > > currently running SparkContext was created at:
> > >
> > > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
> > > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> > >
> > > What I do now is running tests under different packages in hudi-client
> > > manually in IntelliJ. That seems to work for me.
> > >
> > > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org>
> > wrote:
> > >
> > > > Hi Ethan,
> > > >
> > > > For purposes of unblocking yourself, can you try running them locally
> > via
> > > > mvn command via terminal?
> > > >
> > > > thanks
> > > > vinoth
> > > >
> > > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <
> ethan.guoyihua@gmail.com>
> > > > wrote:
> > > >
> > > > > Hi,
> > > > >
> > > > > I'm working on hudi-client module and I notice that if I run all
> unit
> > > > tests
> > > > > under hudi-client locally in IntelliJ, some tests (54 out of 256)
> are
> > > > > failing due to the following SparkException: "Only one SparkContext
> > may
> > > > be
> > > > > running in this JVM". Is there any way I can get around this?
> > > > >
> > > > > org.apache.spark.SparkException: Only one SparkContext may be
> running
> > > in
> > > > > this JVM (see SPARK-2243). To ignore this error, set
> > > > > spark.driver.allowMultipleContexts = true. The currently running
> > > > > SparkContext was created at:
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > > >
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > >
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > > java.lang.reflect.Method.invoke(Method.java:498)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > >
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > > > at scala.Option.foreach(Option.scala:257)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > at
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > at
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > > at
> > > > >
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > > > at
> > > > >
> > > > >
> > > >
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > > > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > > > >
> > > > > Best,
> > > > > - Ethan
> > > > >
> > > >
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Y Ethan Guo <et...@gmail.com>.
I see. I'll check the travis CI setup later. I'm unblocked now for
running the unit tests locally.
Thanks,
- Ethan
On Mon, Jul 27, 2020 at 11:03 PM Vinoth Chandar <vi...@apache.org> wrote:
> So, i realized something. With the recent changes, functional tests retain
> a single spark session for the entire test suite to speed things up. So
> thats probably what you were hitting first, when running via IDE
> Try following the .travis.yml profiles directly?
>
>
> Not sure about the OOM. Have not seen this before.
>
> On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <et...@gmail.com>
> wrote:
>
> > Thanks for the suggestion, Vinoth.
> >
> > I tried a few things on master:
> > - "mvn clean package -DskipITs": It throws the following exception:
> >
> > [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0, Time
> > elapsed: 19.286 s* <<< FAILURE!* - in
> > org.apache.hudi.table.action.rollback.
> > *TestMergeOnReadRollbackActionExecutor*
> >
> > [*ERROR*]
> >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> > Time elapsed: 4.977 s <<< ERROR!
> >
> > org.apache.hudi.exception.HoodieRemoteException: ... failed to respond
> >
> > at
> >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> >
> > Caused by: org.apache.http.NoHttpResponseException: ... failed to respond
> >
> > at
> >
> >
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
> >
> > [*ERROR*]
> >
> >
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> > Time elapsed: 1.182 s <<< ERROR!
> >
> > org.apache.spark.SparkException:
> >
> > Only one SparkContext may be running in this JVM (see SPARK-2243). To
> > ignore this error, set spark.driver.allowMultipleContexts = true. The
> > currently running SparkContext was created at:
> >
> > - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
> > "java.lang.OutOfMemoryError: GC overhead limit exceeded".
> >
> > What I do now is running tests under different packages in hudi-client
> > manually in IntelliJ. That seems to work for me.
> >
> > On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org>
> wrote:
> >
> > > Hi Ethan,
> > >
> > > For purposes of unblocking yourself, can you try running them locally
> via
> > > mvn command via terminal?
> > >
> > > thanks
> > > vinoth
> > >
> > > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <et...@gmail.com>
> > > wrote:
> > >
> > > > Hi,
> > > >
> > > > I'm working on hudi-client module and I notice that if I run all unit
> > > tests
> > > > under hudi-client locally in IntelliJ, some tests (54 out of 256) are
> > > > failing due to the following SparkException: "Only one SparkContext
> may
> > > be
> > > > running in this JVM". Is there any way I can get around this?
> > > >
> > > > org.apache.spark.SparkException: Only one SparkContext may be running
> > in
> > > > this JVM (see SPARK-2243). To ignore this error, set
> > > > spark.driver.allowMultipleContexts = true. The currently running
> > > > SparkContext was created at:
> > > >
> > > >
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > java.lang.reflect.Method.invoke(Method.java:498)
> > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > >
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > > at scala.Option.foreach(Option.scala:257)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > > at
> > > >
> > > >
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > > at
> > > >
> > > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > > at
> > > >
> > > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > at
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > at
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > > at
> > > >
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > > at
> > > >
> > > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > > at
> > > >
> > > >
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > > at
> > > >
> > > >
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > > at
> > > >
> > > >
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > > >
> > > > Best,
> > > > - Ethan
> > > >
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Vinoth Chandar <vi...@apache.org>.
So, i realized something. With the recent changes, functional tests retain
a single spark session for the entire test suite to speed things up. So
thats probably what you were hitting first, when running via IDE
Try following the .travis.yml profiles directly?
Not sure about the OOM. Have not seen this before.
On Mon, Jul 27, 2020 at 11:00 PM Y Ethan Guo <et...@gmail.com>
wrote:
> Thanks for the suggestion, Vinoth.
>
> I tried a few things on master:
> - "mvn clean package -DskipITs": It throws the following exception:
>
> [*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0, Time
> elapsed: 19.286 s* <<< FAILURE!* - in
> org.apache.hudi.table.action.rollback.
> *TestMergeOnReadRollbackActionExecutor*
>
> [*ERROR*]
>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
> Time elapsed: 4.977 s <<< ERROR!
>
> org.apache.hudi.exception.HoodieRemoteException: ... failed to respond
>
> at
>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>
> Caused by: org.apache.http.NoHttpResponseException: ... failed to respond
>
> at
>
> org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
>
> [*ERROR*]
>
> org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
> Time elapsed: 1.182 s <<< ERROR!
>
> org.apache.spark.SparkException:
>
> Only one SparkContext may be running in this JVM (see SPARK-2243). To
> ignore this error, set spark.driver.allowMultipleContexts = true. The
> currently running SparkContext was created at:
>
> - "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
> "java.lang.OutOfMemoryError: GC overhead limit exceeded".
>
> What I do now is running tests under different packages in hudi-client
> manually in IntelliJ. That seems to work for me.
>
> On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org> wrote:
>
> > Hi Ethan,
> >
> > For purposes of unblocking yourself, can you try running them locally via
> > mvn command via terminal?
> >
> > thanks
> > vinoth
> >
> > On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <et...@gmail.com>
> > wrote:
> >
> > > Hi,
> > >
> > > I'm working on hudi-client module and I notice that if I run all unit
> > tests
> > > under hudi-client locally in IntelliJ, some tests (54 out of 256) are
> > > failing due to the following SparkException: "Only one SparkContext may
> > be
> > > running in this JVM". Is there any way I can get around this?
> > >
> > > org.apache.spark.SparkException: Only one SparkContext may be running
> in
> > > this JVM (see SPARK-2243). To ignore this error, set
> > > spark.driver.allowMultipleContexts = true. The currently running
> > > SparkContext was created at:
> > >
> > >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> > >
> > >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > java.lang.reflect.Method.invoke(Method.java:498)
> > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > >
> > > at
> > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > > at
> > >
> > >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > > at scala.Option.foreach(Option.scala:257)
> > > at
> > >
> > >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > > at
> > >
> > >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > > at
> > >
> > >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > > at
> > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > > at
> > >
> > >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > > at
> > >
> > >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > > at
> > >
> > >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > > at
> > >
> > >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > > at java.lang.reflect.Method.invoke(Method.java:498)
> > > at
> > >
> > >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > > at
> > >
> > >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > at
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > at
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > > at
> > >
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > > at
> > >
> > >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > > at
> > >
> > >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > > at
> > >
> > >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > > at
> > >
> > >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > > at
> > >
> > >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> > >
> > > Best,
> > > - Ethan
> > >
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Y Ethan Guo <et...@gmail.com>.
Thanks for the suggestion, Vinoth.
I tried a few things on master:
- "mvn clean package -DskipITs": It throws the following exception:
[*ERROR*] *Tests **run: 3*, Failures: 0, *Errors: 1*, Skipped: 0, Time
elapsed: 19.286 s* <<< FAILURE!* - in org.apache.hudi.table.action.rollback.
*TestMergeOnReadRollbackActionExecutor*
[*ERROR*]
org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(boolean)[2]
Time elapsed: 4.977 s <<< ERROR!
org.apache.hudi.exception.HoodieRemoteException: ... failed to respond
at
org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
Caused by: org.apache.http.NoHttpResponseException: ... failed to respond
at
org.apache.hudi.table.action.rollback.TestMergeOnReadRollbackActionExecutor.testMergeOnReadRollbackActionExecutor(TestMergeOnReadRollbackActionExecutor.java:79)
[*ERROR*]
org.apache.hudi.index.TestHoodieIndex.testSimpleTagLocationAndUpdate(HoodieIndex$IndexType)
Time elapsed: 1.182 s <<< ERROR!
org.apache.spark.SparkException:
Only one SparkContext may be running in this JVM (see SPARK-2243). To
ignore this error, set spark.driver.allowMultipleContexts = true. The
currently running SparkContext was created at:
- "mvn test -Punit-tests -pl hudi-client -B" from CI: It encounters
"java.lang.OutOfMemoryError: GC overhead limit exceeded".
What I do now is running tests under different packages in hudi-client
manually in IntelliJ. That seems to work for me.
On Sun, Jul 26, 2020 at 5:18 PM Vinoth Chandar <vi...@apache.org> wrote:
> Hi Ethan,
>
> For purposes of unblocking yourself, can you try running them locally via
> mvn command via terminal?
>
> thanks
> vinoth
>
> On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <et...@gmail.com>
> wrote:
>
> > Hi,
> >
> > I'm working on hudi-client module and I notice that if I run all unit
> tests
> > under hudi-client locally in IntelliJ, some tests (54 out of 256) are
> > failing due to the following SparkException: "Only one SparkContext may
> be
> > running in this JVM". Is there any way I can get around this?
> >
> > org.apache.spark.SparkException: Only one SparkContext may be running in
> > this JVM (see SPARK-2243). To ignore this error, set
> > spark.driver.allowMultipleContexts = true. The currently running
> > SparkContext was created at:
> >
> >
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
> >
> >
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> > sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > java.lang.reflect.Method.invoke(Method.java:498)
> >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> >
> > at
> >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> > at
> >
> >
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> > at scala.Option.foreach(Option.scala:257)
> > at
> >
> >
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> > at
> >
> >
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> > at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> > at
> >
> >
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> > at
> >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> > at
> >
> >
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> > at
> >
> >
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> > at
> >
> >
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> > at
> >
> >
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> > at java.lang.reflect.Method.invoke(Method.java:498)
> > at
> >
> >
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> > at
> >
> >
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> > at
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> > at
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> > at
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> > at
> >
> >
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> > at
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> > at
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> > at
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> > at
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> > at
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> > at
> >
> >
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> > at
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> > at
> >
> >
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> > at
> >
> >
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > at
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > at
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > at java.util.ArrayList.forEach(ArrayList.java:1249)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> > at
> > org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> > at
> >
> >
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> > at
> >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> > at
> >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> > at
> >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> > at
> >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> > at
> >
> >
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> > at
> >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> > at
> >
> >
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> > at
> >
> >
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> > at
> >
> >
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> > at
> >
> >
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> > at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
> >
> > Best,
> > - Ethan
> >
>
Re: Unit tests in hudi-client module fail due to SparkContext
Posted by Vinoth Chandar <vi...@apache.org>.
Hi Ethan,
For purposes of unblocking yourself, can you try running them locally via
mvn command via terminal?
thanks
vinoth
On Sun, Jul 26, 2020 at 4:12 PM Y Ethan Guo <et...@gmail.com>
wrote:
> Hi,
>
> I'm working on hudi-client module and I notice that if I run all unit tests
> under hudi-client locally in IntelliJ, some tests (54 out of 256) are
> failing due to the following SparkException: "Only one SparkContext may be
> running in this JVM". Is there any way I can get around this?
>
> org.apache.spark.SparkException: Only one SparkContext may be running in
> this JVM (see SPARK-2243). To ignore this error, set
> spark.driver.allowMultipleContexts = true. The currently running
> SparkContext was created at:
>
> org.apache.spark.sql.SparkSession$Builder.getOrCreate(SparkSession.scala:926)
>
> org.apache.hudi.testutils.FunctionalTestHarness.runBeforeEach(FunctionalTestHarness.java:132)
> sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> java.lang.reflect.Method.invoke(Method.java:498)
>
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
>
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
>
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
>
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
>
> at
>
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2483)
> at
>
> org.apache.spark.SparkContext$$anonfun$assertNoOtherContextIsRunning$2.apply(SparkContext.scala:2479)
> at scala.Option.foreach(Option.scala:257)
> at
>
> org.apache.spark.SparkContext$.assertNoOtherContextIsRunning(SparkContext.scala:2479)
> at
>
> org.apache.spark.SparkContext$.markPartiallyConstructed(SparkContext.scala:2568)
> at org.apache.spark.SparkContext.<init>(SparkContext.scala:85)
> at
>
> org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
> at
>
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:126)
> at
>
> org.apache.hudi.testutils.HoodieClientTestHarness.initSparkContexts(HoodieClientTestHarness.java:139)
> at
>
> org.apache.hudi.index.bloom.TestHoodieGlobalBloomIndex.setUp(TestHoodieGlobalBloomIndex.java:74)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
>
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
>
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
>
> org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:688)
> at
>
> org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60)
> at
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131)
> at
>
> org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:149)
> at
>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptLifecycleMethod(TimeoutExtension.java:126)
> at
>
> org.junit.jupiter.engine.extension.TimeoutExtension.interceptBeforeEachMethod(TimeoutExtension.java:76)
> at
>
> org.junit.jupiter.engine.execution.ExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(ExecutableInvoker.java:115)
> at
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.lambda$invoke$0(ExecutableInvoker.java:105)
> at
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106)
> at
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64)
> at
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45)
> at
>
> org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37)
> at
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:104)
> at
>
> org.junit.jupiter.engine.execution.ExecutableInvoker.invoke(ExecutableInvoker.java:98)
> at
>
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.invokeMethodInExtensionContext(ClassBasedTestDescriptor.java:481)
> at
>
> org.junit.jupiter.engine.descriptor.ClassBasedTestDescriptor.lambda$synthesizeBeforeEachMethodAdapter$18(ClassBasedTestDescriptor.java:466)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeEachMethods$2(TestMethodTestDescriptor.java:169)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeBeforeMethodsOrCallbacksUntilExceptionOccurs$5(TestMethodTestDescriptor.java:197)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeMethodsOrCallbacksUntilExceptionOccurs(TestMethodTestDescriptor.java:197)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeBeforeEachMethods(TestMethodTestDescriptor.java:166)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:133)
> at
>
> org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:71)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:139)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> at java.util.ArrayList.forEach(ArrayList.java:1249)
> at
>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> at java.util.ArrayList.forEach(ArrayList.java:1249)
> at
>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:38)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$5(NodeTestTask.java:143)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$7(NodeTestTask.java:129)
> at
> org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:127)
> at
>
> org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:126)
> at
>
> org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:84)
> at
>
> org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:32)
> at
>
> org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57)
> at
>
> org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:51)
> at
>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:107)
> at
>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:87)
> at
>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:53)
> at
>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:66)
> at
>
> org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:51)
> at
>
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:87)
> at
>
> org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:66)
> at
>
> com.intellij.junit5.JUnit5IdeaTestRunner.startRunnerWithArgs(JUnit5IdeaTestRunner.java:69)
> at
>
> com.intellij.rt.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:33)
> at
>
> com.intellij.rt.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:230)
> at com.intellij.rt.junit.JUnitStarter.main(JUnitStarter.java:58)
>
> Best,
> - Ethan
>