You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by GitBox <gi...@apache.org> on 2020/10/21 19:35:48 UTC
[GitHub] [druid] technomage commented on issue #10523: Historical server fails to load segments in kubernetes
technomage commented on issue #10523:
URL: https://github.com/apache/druid/issues/10523#issuecomment-713828041
I applied the changes recommended for storage config and I am still seeing similar errors. Here is the log, full values to follow.
search
Search
Workloads
keyboard_arrow_right
Pods
keyboard_arrow_right
cog-druid-historical-0
keyboard_arrow_right
Logs
Namespace
default
Logs from
druid
in
cog-druid-historical-0
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegment(SegmentLoaderLocalCacheManager.java:129) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.getAdapter(SegmentManager.java:218) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.loadSegment(SegmentManager.java:177) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.loadSegment(SegmentLoadDropHandler.java:265) ~[druid-server-0.19.0.jar:0.19.0]
... 8 more
2020-10-21T19:32:52,646 INFO [ZkCoordinator] org.apache.druid.server.coordination.ZkCoordinator - zNode[/druid/loadQueue/172.17.0.29:8083/people_1972-01-01T00:00:00.000Z_1973-01-01T00:00:00.000Z_2020-10-21T18:30:30.853Z] was removed
2020-10-21T19:32:52,646 INFO [ZKCoordinator--0] org.apache.druid.server.coordination.ZkCoordinator - Completed request [LOAD: people_1972-01-01T00:00:00.000Z_1973-01-01T00:00:00.000Z_2020-10-21T18:30:30.853Z]
2020-10-21T19:32:52,650 INFO [ZKCoordinator--0] org.apache.druid.server.coordination.SegmentLoadDropHandler - Loading segment people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z
2020-10-21T19:32:52,650 WARN [ZKCoordinator--0] org.apache.druid.server.coordination.BatchDataSegmentAnnouncer - No path to unannounce segment[people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z]
2020-10-21T19:32:52,650 INFO [ZKCoordinator--0] org.apache.druid.server.SegmentManager - Told to delete a queryable for a dataSource[people] that doesn't exist.
2020-10-21T19:32:52,651 INFO [ZKCoordinator--0] org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager - Deleting directory[/opt/druid/var/druid/segment-cache/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z/0]
2020-10-21T19:32:52,651 INFO [ZKCoordinator--0] org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager - Deleting directory[/opt/druid/var/druid/segment-cache/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z]
2020-10-21T19:32:52,651 INFO [ZKCoordinator--0] org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager - Deleting directory[/opt/druid/var/druid/segment-cache/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z]
2020-10-21T19:32:52,652 INFO [ZKCoordinator--0] org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager - Deleting directory[/opt/druid/var/druid/segment-cache/people]
2020-10-21T19:32:52,652 WARN [ZKCoordinator--0] org.apache.druid.server.coordination.SegmentLoadDropHandler - Unable to delete segmentInfoCacheFile[/opt/druid/var/druid/info/people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z]
2020-10-21T19:32:52,652 ERROR [ZKCoordinator--0] org.apache.druid.server.coordination.SegmentLoadDropHandler - Failed to load segment for dataSource: {class=org.apache.druid.server.coordination.SegmentLoadDropHandler, exceptionType=class org.apache.druid.segment.loading.SegmentLoadingException, exceptionMessage=Exception loading segment[people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z], segment=DataSegment{binaryVersion=9, id=people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z, loadSpec={type=>local, path=>/opt/druid/var/druid/storage/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z/0/be3f658b-e69a-4451-9fad-1ab0e7f9d577/index.zip}, dimensions=[birthdate, blood_type, first_name, id, last_name, name, role], metrics=[count], shardSpec=NumberedShardSpec{partitionNum=0, partitions=0}, lastCompactionState=null, size=28754336}}
org.apache.druid.segment.loading.SegmentLoadingException: Exception loading segment[people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.loadSegment(SegmentLoadDropHandler.java:269) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.addSegment(SegmentLoadDropHandler.java:313) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentChangeRequestLoad.go(SegmentChangeRequestLoad.java:61) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.ZkCoordinator.lambda$childAdded$2(ZkCoordinator.java:147) ~[druid-server-0.19.0.jar:0.19.0]
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) [?:1.8.0_252]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_252]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_252]
Caused by: java.lang.IllegalArgumentException: Cannot construct instance of `org.apache.druid.segment.loading.LocalLoadSpec`, problem: [/opt/druid/var/druid/storage/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z/0/be3f658b-e69a-4451-9fad-1ab0e7f9d577/index.zip] does not exist
at [Source: UNKNOWN; line: -1, column: -1]
at com.fasterxml.jackson.databind.ObjectMapper._convert(ObjectMapper.java:3922) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.ObjectMapper.convertValue(ObjectMapper.java:3853) ~[jackson-databind-2.10.2.jar:2.10.2]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocation(SegmentLoaderLocalCacheManager.java:240) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocationWithStartMarker(SegmentLoaderLocalCacheManager.java:229) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadSegmentWithRetry(SegmentLoaderLocalCacheManager.java:190) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegmentFiles(SegmentLoaderLocalCacheManager.java:162) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegment(SegmentLoaderLocalCacheManager.java:129) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.getAdapter(SegmentManager.java:218) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.loadSegment(SegmentManager.java:177) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.loadSegment(SegmentLoadDropHandler.java:265) ~[druid-server-0.19.0.jar:0.19.0]
... 8 more
Caused by: com.fasterxml.jackson.databind.exc.ValueInstantiationException: Cannot construct instance of `org.apache.druid.segment.loading.LocalLoadSpec`, problem: [/opt/druid/var/druid/storage/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z/0/be3f658b-e69a-4451-9fad-1ab0e7f9d577/index.zip] does not exist
at [Source: UNKNOWN; line: -1, column: -1]
at com.fasterxml.jackson.databind.exc.ValueInstantiationException.from(ValueInstantiationException.java:47) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.DeserializationContext.instantiationException(DeserializationContext.java:1732) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.wrapAsJsonMappingException(StdValueInstantiator.java:491) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.rewrapCtorProblem(StdValueInstantiator.java:514) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.createFromObjectWith(StdValueInstantiator.java:285) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.ValueInstantiator.createFromObjectWith(ValueInstantiator.java:229) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.impl.PropertyBasedCreator.build(PropertyBasedCreator.java:198) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:488) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1287) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:326) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther(BeanDeserializer.java:194) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:161) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId(AsPropertyTypeDeserializer.java:130) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject(AsPropertyTypeDeserializer.java:97) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType(AbstractDeserializer.java:254) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer.deserialize(TypeWrappedDeserializer.java:68) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.ObjectMapper._convert(ObjectMapper.java:3917) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.ObjectMapper.convertValue(ObjectMapper.java:3853) ~[jackson-databind-2.10.2.jar:2.10.2]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocation(SegmentLoaderLocalCacheManager.java:240) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocationWithStartMarker(SegmentLoaderLocalCacheManager.java:229) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadSegmentWithRetry(SegmentLoaderLocalCacheManager.java:190) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegmentFiles(SegmentLoaderLocalCacheManager.java:162) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegment(SegmentLoaderLocalCacheManager.java:129) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.getAdapter(SegmentManager.java:218) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.loadSegment(SegmentManager.java:177) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.loadSegment(SegmentLoadDropHandler.java:265) ~[druid-server-0.19.0.jar:0.19.0]
... 8 more
Caused by: java.lang.IllegalArgumentException: [/opt/druid/var/druid/storage/people/1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z/2020-10-21T18:30:31.071Z/0/be3f658b-e69a-4451-9fad-1ab0e7f9d577/index.zip] does not exist
at com.google.common.base.Preconditions.checkArgument(Preconditions.java:148) ~[guava-16.0.1.jar:?]
at org.apache.druid.segment.loading.LocalLoadSpec.<init>(LocalLoadSpec.java:51) ~[druid-server-0.19.0.jar:0.19.0]
at sun.reflect.GeneratedConstructorAccessor58.newInstance(Unknown Source) ~[?:?]
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_252]
at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_252]
at com.fasterxml.jackson.databind.introspect.AnnotatedConstructor.call(AnnotatedConstructor.java:124) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.std.StdValueInstantiator.createFromObjectWith(StdValueInstantiator.java:283) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.ValueInstantiator.createFromObjectWith(ValueInstantiator.java:229) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.impl.PropertyBasedCreator.build(PropertyBasedCreator.java:198) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:488) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1287) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:326) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeOther(BeanDeserializer.java:194) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:161) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer._deserializeTypedForId(AsPropertyTypeDeserializer.java:130) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.jsontype.impl.AsPropertyTypeDeserializer.deserializeTypedFromObject(AsPropertyTypeDeserializer.java:97) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.AbstractDeserializer.deserializeWithType(AbstractDeserializer.java:254) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.deser.impl.TypeWrappedDeserializer.deserialize(TypeWrappedDeserializer.java:68) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.ObjectMapper._convert(ObjectMapper.java:3917) ~[jackson-databind-2.10.2.jar:2.10.2]
at com.fasterxml.jackson.databind.ObjectMapper.convertValue(ObjectMapper.java:3853) ~[jackson-databind-2.10.2.jar:2.10.2]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocation(SegmentLoaderLocalCacheManager.java:240) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadInLocationWithStartMarker(SegmentLoaderLocalCacheManager.java:229) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.loadSegmentWithRetry(SegmentLoaderLocalCacheManager.java:190) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegmentFiles(SegmentLoaderLocalCacheManager.java:162) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.segment.loading.SegmentLoaderLocalCacheManager.getSegment(SegmentLoaderLocalCacheManager.java:129) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.getAdapter(SegmentManager.java:218) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.SegmentManager.loadSegment(SegmentManager.java:177) ~[druid-server-0.19.0.jar:0.19.0]
at org.apache.druid.server.coordination.SegmentLoadDropHandler.loadSegment(SegmentLoadDropHandler.java:265) ~[druid-server-0.19.0.jar:0.19.0]
... 8 more
2020-10-21T19:32:52,664 INFO [ZKCoordinator--0] org.apache.druid.server.coordination.ZkCoordinator - Completed request [LOAD: people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z]
2020-10-21T19:32:52,665 INFO [ZkCoordinator] org.apache.druid.server.coordination.ZkCoordinator - zNode[/druid/loadQueue/172.17.0.29:8083/people_1971-01-01T00:00:00.000Z_1972-01-01T00:00:00.000Z_2020-10-21T18:30:31.071Z] was removed
Logs from Oct 21, 2020 to Oct 21, 2020 UTC
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org