You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@carbondata.apache.org by "Indhumathi Muthu Murugesh (Jira)" <ji...@apache.org> on 2021/02/04 10:02:00 UTC
[jira] [Resolved] (CARBONDATA-4111) Filter query having invalid
results after add segment to table having SI with Indexserver
[ https://issues.apache.org/jira/browse/CARBONDATA-4111?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Indhumathi Muthu Murugesh resolved CARBONDATA-4111.
---------------------------------------------------
Fix Version/s: 2.2.0
Resolution: Fixed
> Filter query having invalid results after add segment to table having SI with Indexserver
> -----------------------------------------------------------------------------------------
>
> Key: CARBONDATA-4111
> URL: https://issues.apache.org/jira/browse/CARBONDATA-4111
> Project: CarbonData
> Issue Type: Bug
> Reporter: SHREELEKHYA GAMPA
> Priority: Minor
> Fix For: 2.2.0
>
> Attachments: addseg_si_is.png
>
> Time Spent: 7h 10m
> Remaining Estimate: 0h
>
> queries to execute:
> create table maintable_sdk(a string, b int, c string) stored as carbondata;
> insert into maintable_sdk select 'k',1,'k';
> insert into maintable_sdk select 'l',2,'l';
> CREATE INDEX maintable_si_sdk on table maintable_sdk (c) as 'carbondata';
> alter table maintable_sdk add segment options('path'='hdfs://hacluster/sdkfiles/newsegment/', 'format'='carbon');
> spark-sql> select *from maintable_sdk where c='m';
> 2021-01-27 12:10:54,326 | WARN | IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from carbon@HADOOP.COM | Unexpected error reading responses on connection Thread[IPC Client (653337757) connection to linux-30/10.19.90.30:22900 from carbon@HADOOP.COM,5,main] | org.apache.hadoop.ipc.Client.run(Client.java:1113)
> java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
> at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
> at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
> at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
> at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
> at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
> at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
> at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
> at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
> at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
> Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
> at java.lang.Class.getConstructor0(Class.java:3082)
> at java.lang.Class.getDeclaredConstructor(Class.java:2178)
> at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
> ... 8 more
> 2021-01-27 12:10:54,330 | WARN | main | Distributed Segment Pruning failed, initiating embedded pruning | org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:349)
> java.lang.reflect.UndeclaredThrowableException
> at com.sun.proxy.$Proxy59.getPrunedSegments(Unknown Source)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:341)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin$.getFilteredSegments(BroadCastSIFilterPushJoin.scala:426)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions$lzycompute(BroadCastSIFilterPushJoin.scala:80)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.partitions(BroadCastSIFilterPushJoin.scala:78)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy$lzycompute(BroadCastSIFilterPushJoin.scala:94)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.inputCopy(BroadCastSIFilterPushJoin.scala:93)
> at org.apache.spark.sql.secondaryindex.joins.BroadCastSIFilterPushJoin.doExecute(BroadCastSIFilterPushJoin.scala:132)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:177)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:173)
> at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:201)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:198)
> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:173)
> at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:293)
> at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:342)
> at org.apache.spark.sql.execution.SparkPlan.executeCollectPublic(SparkPlan.scala:372)
> at org.apache.spark.sql.execution.QueryExecution.hiveResultString(QueryExecution.scala:127)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver$$anonfun$run$1.apply(SparkSQLDriver.scala:66)
> at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1$$anonfun$apply$1.apply(SQLExecution.scala:95)
> at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:144)
> at org.apache.spark.sql.execution.SQLExecution$$anonfun$withNewExecutionId$1.apply(SQLExecution.scala:86)
> at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:789)
> at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:63)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLDriver.run(SparkSQLDriver.scala:65)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.processCmd(SparkSQLCLIDriver.scala:383)
> at org.apache.hadoop.hive.cli.CliDriver.processLine(CliDriver.java:406)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver$.main(SparkSQLCLIDriver.scala:277)
> at org.apache.spark.sql.hive.thriftserver.SparkSQLCLIDriver.main(SparkSQLCLIDriver.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:882)
> at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:164)
> at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:187)
> at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:89)
> at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:957)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:966)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> Caused by: java.io.IOException: DestHost:destPort linux-30:22900 , LocalHost:localPort linux-28/10.19.90.28:0. Failed on local exception: java.io.IOException: Error reading responses
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
> at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:1014)
> at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:989)
> at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1567)
> at org.apache.hadoop.ipc.Client.call(Client.java:1509)
> at org.apache.hadoop.ipc.Client.call(Client.java:1408)
> at org.apache.hadoop.ipc.WritableRpcEngine$Invoker.invoke(WritableRpcEngine.java:251)
> ... 42 more
> Caused by: java.io.IOException: Error reading responses
> at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1114)
> Caused by: java.lang.RuntimeException: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
> at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:135)
> at org.apache.hadoop.io.WritableFactories.newInstance(WritableFactories.java:58)
> at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:284)
> at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
> at org.apache.hadoop.ipc.RpcWritable$WritableWrapper.readFrom(RpcWritable.java:85)
> at org.apache.hadoop.ipc.RpcWritable$Buffer.getValue(RpcWritable.java:187)
> at org.apache.hadoop.ipc.RpcWritable$Buffer.newInstance(RpcWritable.java:183)
> at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1223)
> at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1107)
> Caused by: java.lang.NoSuchMethodException: org.apache.carbondata.core.indexstore.SegmentWrapperContainer.<init>()
> at java.lang.Class.getConstructor0(Class.java:3082)
> at java.lang.Class.getDeclaredConstructor(Class.java:2178)
> at org.apache.hadoop.util.ReflectionUtils.newInstance(ReflectionUtils.java:129)
> ... 8 more
> Time taken: 2.813 seconds
> spark-sql> select *from maintable_sdk;
> k 1 k
> l 2 l
> m 3 m
> Time taken: 0.569 seconds, Fetched 3 row(s)
--
This message was sent by Atlassian Jira
(v8.3.4#803005)