[GitHub] [pinot] saurabhd336 commented on pull request #11158: Increment nextDocId even if geo indexing fails

["saurabhd336 (via GitHub)" <gi...@apache.org>]

saurabhd336 commented on PR #11158:
URL: https://github.com/apache/pinot/pull/11158#issuecomment-1647784240

   Issue reproducible with
   
   Schema
   ```
   {
     "schemaName": "geoIndex",
     "dimensionFieldSpecs": [
       {
         "name": "geoIndexCol",
         "dataType": "BYTES"
       },
       {
         "name": "lon",
         "dataType": "STRING"
       },
       {
         "name": "lat",
         "dataType": "STRING"
       }
     ],
     "dateTimeFieldSpecs": [{
       "name": "timestampInEpoch",
       "dataType": "LONG",
       "format" : "1:MILLISECONDS:EPOCH",
       "granularity": "1:MILLISECONDS"
     }]
   }
   ```
   
   TableConfig
   ```
   {
   	"tableName": "geoTest",
   	"tableType": "REALTIME",
   	"segmentsConfig": {
   		"timeColumnName": "timestampInEpoch",
   		"timeType": "MILLISECONDS",
   		"schemaName": "geoIndex",
   		"replicasPerPartition": "1"
   	},
   	"fieldConfigList": [{
   		"name": "geoIndexCol",
   		"encodingType": "RAW",
   		"indexType": "H3",
   		"indexTypes": [
   			"H3"
   		],
   		"properties": {
   			"resolutions": "5"
   		},
   		"tierOverwrites": null
   	}],
   	"tenants": {},
   	"tableIndexConfig": {
   		"loadMode": "MMAP",
   		"streamConfigs": {
   			"streamType": "kafka",
   			"stream.kafka.consumer.type": "lowlevel",
   			"stream.kafka.topic.name": "geo-test-topic",
   			"stream.kafka.decoder.class.name": "org.apache.pinot.plugin.stream.kafka.KafkaJSONMessageDecoder",
   			"stream.kafka.consumer.factory.class.name": "org.apache.pinot.plugin.stream.kafka20.KafkaConsumerFactory",
   			"stream.kafka.broker.list": "localhost:19092",
   			"realtime.segment.flush.threshold.rows": "20",
   			"realtime.segment.flush.threshold.time": "10m",
   			"realtime.segment.flush.threshold.segment.size": "50M",
   			"stream.kafka.consumer.prop.auto.offset.reset": "smallest"
   		}
   	},
   	"ingestionConfig": {
   		"transformConfigs": [{
   			"columnName": "geoIndexCol",
   			"transformFunction": "toSphericalGeography(stPoint(lon,lat))"
   		}]
   	},
   	"metadata": {
   		"customConfigs": {}
   	}
   }
   ```
   
   
   data.json
   ```
   {"lat":"12.933540687183736","lon":"77.62542483501983","timestampInEpoch":1690192996000}
   {"lat":null, "lon":"77.62542483501983","timestampInEpoch":1690192996000}
   {"lat":null, "lon":"77.62542483501983","timestampInEpoch":1690192996000}
   {"lat":"12.933540687183736","lon":"77.62542483501983","timestampInEpoch":1690192996000}
   ```
   
   Query
   ```
   SELECT * FROM geoTest where  ST_DISTANCE(locationSTPoint, ST_Point(77.62357943770779, 12.933540687183736, 1)) < 500
   ```
   
   Fails with exception without this fix since the mutable index has incorrect docIds in the h3Id -> bitmap mapping.
   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pinot.apache.org
For additional commands, e-mail: commits-help@pinot.apache.org