You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pinot.apache.org by GitBox <gi...@apache.org> on 2019/02/25 07:42:26 UTC
[GitHub] ly923976094 edited a comment on issue #3454: what is the difference
between mmap and heap?
ly923976094 edited a comment on issue #3454: what is the difference between mmap and heap?
URL: https://github.com/apache/incubator-pinot/issues/3454#issuecomment-466903049
> Can you give more details on qps, schema and metadata of any segment that was successfully created?
> […](#)
> On Sun, Feb 24, 2019 at 9:58 PM Sunny ***@***.***> wrote: @ly923976094 <https://github.com/ly923976094> that depends on your hardware profile, OS, qps load, latency tolerance, and amount of data. It is a very hard question to answer, but we plan to have some write-ups on tuning Pinot. It should be available with our documentation in the next few weeks. Is there any specific document for reference? At present, consumers with high QPS have configured HEAP and often out memory @mcvsubbu <https://github.com/mcvsubbu> — You are receiving this because you commented. Reply to this email directly, view it on GitHub <[#3454 (comment)](https://github.com/apache/incubator-pinot/issues/3454#issuecomment-466882031)>, or mute the thread <https://github.com/notifications/unsubscribe-auth/AAz7Z35iZRrC1kzRYt9x5Xi-hHXLwySxks5vQ3uVgaJpZM4YWFP7> .
Thank you for your reply, When the consumer is consuming, when the state is transitioning, memory out of memory, consumption stops. What's going on here
server configuration（6+32G, jvm16G, consumer qps 5000,
tablle config:
{
"REALTIME": {
"tableType": "REALTIME",
"tenants": {
"server": "serverOne",
"tagOverrideConfig": null,
"broker": "brokerOne"
},
"tableIndexConfig": {
"onHeapDictionaryColumns": null,
"autoGeneratedInvertedIndex": false,
"sortedColumn": [],
"starTreeIndexConfigs": null,
"invertedIndexColumns": null,
"segmentFormatVersion": null,
"noDictionaryConfig": null,
"createInvertedIndexDuringSegmentGeneration": false,
"starTreeIndexSpec": null,
"streamConfigs": {
"stream.kafka.decoder.class.name": "com.linkedin.pinot.core.realtime.impl.kafka.KafkaJSONMessageDecoder",
"streamType": "kafka",
"realtime.segment.flush.threshold.size": "5000000",
"stream.kafka.consumer.type": "simple",
"stream.kafka.broker.list": " broker1.intra.aliyun.dip.weibo.com:9090,broker2.intra.aliyun.dip.weibo.com:9090,broker3.intra.aliyun.dip.weibo.com:9090",
"realtime.segment.flush.threshold.time": "3600000",
"stream.kafka.consumer.prop.auto.offset.reset": "smallest",
"stream.kafka.hlc.group.id": "dip_cdn_client-video_aggregation",
"stream.kafka.topic.name": "cdn_client"
},
"loadMode": "HEAP",
"columnMinMaxValueGeneratorMode": null,
"noDictionaryColumns": null,
"segmentPartitionConfig": null,
"streamConsumptionConfig": null,
"aggregateMetrics": false
},
"metadata": {
"customConfigs": {
"business": "dip-cdn-client-video-aggregation",
"isFromEsKafka": "true"
}
},
"segmentsConfig": {
"retentionTimeUnit": "DAYS",
"segmentPushFrequency": "daily",
"replication": "1",
"replicaGroupStrategyConfig": null,
"timeColumnName": "fdate",
"retentionTimeValue": "4",
"timeType": "MILLISECONDS",
"segmentPushType": "APPEND",
"hllConfig": null,
"schemaName": "dip_cdn_client-video_aggregation",
"segmentAssignmentStrategy": "BalanceNumSegmentAssignmentStrategy",
"replicasPerPartition": "1"
},
"tableName": "dip_cdn_client-video_aggregation_REALTIME"
}
}
schema config:
{
"schemaName": "dip_cdn_client-video_aggregation",
"dimensionFieldSpecs": [
{
"name": "cdn",
"dataType": "STRING"
},
{
"name": "country",
"dataType": "STRING"
},
{
"name": "dns_ip_country",
"dataType": "STRING"
},
{
"name": "dns_ip_isp",
"dataType": "STRING"
},
{
"name": "dns_ip_province",
"dataType": "STRING"
},
{
"name": "domain",
"dataType": "STRING"
},
{
"name": "error_code",
"dataType": "STRING"
},
{
"name": "isp",
"dataType": "STRING"
},
{
"name": "protocal",
"dataType": "STRING"
},
{
"name": "protocol",
"dataType": "STRING"
},
{
"name": "province",
"dataType": "STRING"
},
{
"name": "video_cache_type",
"dataType": "STRING"
},
{
"name": "video_firstframe_status",
"dataType": "STRING"
},
{
"name": "video_network",
"dataType": "STRING"
},
{
"name": "video_quit_status",
"dataType": "STRING"
},
{
"name": "video_trace_dns_ip",
"dataType": "STRING"
}
],
"metricFieldSpecs": [
{
"name": "ff_cancel_num",
"dataType": "LONG"
},
{
"name": "ff_error_num",
"dataType": "LONG"
},
{
"name": "psr1_denominator",
"dataType": "LONG"
},
{
"name": "psr1_numerator",
"dataType": "LONG"
},
{
"name": "stall_count",
"dataType": "LONG"
},
{
"name": "succ_denominator",
"dataType": "LONG"
},
{
"name": "total_count",
"dataType": "LONG"
},
{
"name": "total_num1",
"dataType": "LONG"
},
{
"name": "total_num2",
"dataType": "LONG"
},
{
"name": "succ_numerator",
"dataType": "LONG"
},
{
"name": "video_firstframe_time",
"dataType": "DOUBLE"
}
],
"timeFieldSpec": {
"incomingGranularitySpec": {
"name": "fdate",
"dataType": "LONG",
"timeType": "MILLISECONDS"
}
}
}
java.lang.reflect.InvocationTargetException
at sun.reflect.GeneratedMethodAccessor50.invoke(Unknown Source)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.helix.messaging.handling.HelixStateTransitionHandler.invoke(HelixStateTransitionHandler.java:430)
at org.apache.helix.messaging.handling.HelixStateTransitionHandler.handleMessage(HelixStateTransitionHandler.java:344)
at org.apache.helix.messaging.handling.HelixTask.call(HelixTask.java:96)
at org.apache.helix.messaging.handling.HelixTask.call(HelixTask.java:47)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.OutOfMemoryError: Direct buffer memory
at java.nio.Bits.reserveMemory(Bits.java:693)
at java.nio.DirectByteBuffer.<init>(DirectByteBuffer.java:123)
at java.nio.ByteBuffer.allocateDirect(ByteBuffer.java:311)
at com.linkedin.pinot.core.segment.memory.PinotByteBuffer.allocateDirect(PinotByteBuffer.java:36)
at com.linkedin.pinot.core.segment.memory.PinotByteBuffer.loadFile(PinotByteBuffer.java:40)
at com.linkedin.pinot.core.segment.memory.PinotDataBuffer.loadFile(PinotDataBuffer.java:144)
at com.linkedin.pinot.core.segment.store.SingleFileIndexDirectory.mapAndSliceFile(SingleFileIndexDirectory.java:284)
at com.linkedin.pinot.core.segment.store.SingleFileIndexDirectory.mapBufferEntries(SingleFileIndexDirectory.java:265)
at com.linkedin.pinot.core.segment.store.SingleFileIndexDirectory.load(SingleFileIndexDirectory.java:190)
at com.linkedin.pinot.core.segment.store.SingleFileIndexDirectory.<init>(SingleFileIndexDirectory.java:86)
at com.linkedin.pinot.core.segment.store.SegmentLocalFSDirectory.loadData(SegmentLocalFSDirectory.java:198)
at com.linkedin.pinot.core.segment.store.SegmentLocalFSDirectory.load(SegmentLocalFSDirectory.java:179)
at com.linkedin.pinot.core.segment.store.SegmentLocalFSDirectory.<init>(SegmentLocalFSDirectory.java:77)
at com.linkedin.pinot.core.segment.store.SegmentDirectory.createFromLocalFS(SegmentDirectory.java:108)
at com.linkedin.pinot.core.indexsegment.immutable.ImmutableSegmentLoader.load(ImmutableSegmentLoader.java:108)
at com.linkedin.pinot.core.data.manager.realtime.RealtimeTableDataManager.replaceLLSegment(RealtimeTableDataManager.java:291)
at com.linkedin.pinot.core.data.manager.realtime.RealtimeTableDataManager.downloadAndReplaceSegment(RealtimeTableDataManager.java:279)
at com.linkedin.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.downloadSegmentAndReplace(LLRealtimeSegmentDataManager.java:945)
at com.linkedin.pinot.core.data.manager.realtime.LLRealtimeSegmentDataManager.goOnlineFromConsuming(LLRealtimeSegmentDataManager.java:934)
at com.linkedin.pinot.server.starter.helix.SegmentOnlineOfflineStateModelFactory$SegmentOnlineOfflineStateModel.onBecomeOnlineFromConsuming(SegmentOnlineOfflineStateModelFactory.java:118)
... 11 more
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org