[GitHub] [druid] Techops-ather opened a new issue #11990: My kafka ingestion supervisor is running and tasks also running and getting successed, but I couldnt see any datasource for the ingestion

[GitBox <gi...@apache.org>]

Techops-ather opened a new issue #11990:
URL: https://github.com/apache/druid/issues/11990


   Please find below the spec for the supervisor,
   
   {
     "type": "kafka",
     "spec": {
       "dataSchema": {
         "dataSource": "RawData",
         "timestampSpec": {
           "column": "timestamp",
           "format": "auto",
         },
         "dimensionsSpec": {
           "dimensions": [
             {
               "type": "string",
               "name": "column1",
               "multiValueHandling": "SORTED_ARRAY",
               "createBitmapIndex": true
             },
             {
               "type": "string",
               "name": "column2",
               "multiValueHandling": "SORTED_ARRAY",
               "createBitmapIndex": true
             },
             {
               "type": "string",
               "name": "column3",
               "multiValueHandling": "SORTED_ARRAY",
               "createBitmapIndex": true
             },
             {
               "type": "string",
               "name": "value",
               "multiValueHandling": "SORTED_ARRAY",
               "createBitmapIndex": true
             }
           ],
           "dimensionExclusions": [
             "timestamp"
           ]
         },
         "metricsSpec": [],
         "granularitySpec": {
           "type": "uniform",
           "segmentGranularity": "HOUR",
           "queryGranularity": {
             "type": "none"
           },
           "rollup": false,
           "intervals": null
         },
         "transformSpec": {
           "filter": null,
           "transforms": []
         }
       },
       "ioConfig": {
         "topic": "connect-archive-ather-ather",
         "inputFormat": {
           "type": "json",
           "flattenSpec": {
             "useFieldDiscovery": true,
             "fields": []
           },
           "featureSpec": {}
         },
         "replicas": 2,
         "taskCount": 5,
         "taskDuration": "PT3600S",
         "consumerProperties": {
           "bootstrap.servers": "ap-connect-kafka.ather.vimana.us:9094"
         },
         "pollTimeout": 100,
         "startDelay": "PT5S",
         "period": "PT30S",
         "useEarliestOffset": true,
         "completionTimeout": "PT1800S",
         "lateMessageRejectionPeriod": null,
         "earlyMessageRejectionPeriod": null,
         "lateMessageRejectionStartDateTime": null,
         "stream": "connect-archive-ather-ather",
         "useEarliestSequenceNumber": true,
         "type": "kafka"
       },
       "tuningConfig": {
         "type": "kafka",
         "maxRowsInMemory": 1000000,
         "maxBytesInMemory": 0,
         "maxRowsPerSegment": 5000000,
         "maxTotalRows": null,
         "intermediatePersistPeriod": "PT10M",
         "basePersistDirectory": "/tmp/druid-realtime-persist5702872722921317390",
         "maxPendingPersists": 0,
         "indexSpec": {
           "bitmap": {
             "type": "roaring",
             "compressRunOnSerialization": true
           },
           "dimensionCompression": "lz4",
           "metricCompression": "lz4",
           "longEncoding": "longs",
           "segmentLoader": null
         },
         "indexSpecForIntermediatePersists": {
           "bitmap": {
             "type": "roaring",
             "compressRunOnSerialization": true
           },
           "dimensionCompression": "lz4",
           "metricCompression": "lz4",
           "longEncoding": "longs",
           "segmentLoader": null
         },
         "buildV9Directly": true,
         "reportParseExceptions": false,
         "handoffConditionTimeout": 0,
         "resetOffsetAutomatically": false,
         "segmentWriteOutMediumFactory": null,
         "workerThreads": null,
         "chatThreads": null,
         "chatRetries": 8,
         "httpTimeout": "PT10S",
         "shutdownTimeout": "PT80S",
         "offsetFetchPeriod": "PT30S",
         "intermediateHandoffPeriod": "P2147483647D",
         "logParseExceptions": false,
         "maxParseExceptions": 2147483647,
         "maxSavedParseExceptions": 0,
         "skipSequenceNumberAvailabilityCheck": false,
         "repartitionTransitionDuration": "PT120S"
       }
     }
   }
   
   I think something to do with the timestamp but not sure what to do I have tried with different variations but couldnt able to solve the issue...


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org

[GitHub] [druid] Techops-ather closed issue #11990: My kafka ingestion supervisor is running and tasks also running and getting successed, but I couldnt see any datasource for the ingestion

[GitBox <gi...@apache.org>]

Techops-ather closed issue #11990:
URL: https://github.com/apache/druid/issues/11990


   


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org