You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by "robycuccurese (via GitHub)" <gi...@apache.org> on 2024/05/23 13:52:45 UTC

Re: [I] Sampler with Inline input source swallows up columns that are all `null` (druid)

robycuccurese commented on issue #9658:
URL: https://github.com/apache/druid/issues/9658#issuecomment-2127178016

   Hi,
   
   I try to submit this task:
   
   `{
     "type": "index",
     "spec": {
       "dataSchema": {
         "dataSource": "sample",
         "timestampSpec": {
           "column": "!!!_no_such_column_!!!",
           "format": "auto",
           "missingValue": "1970-01-01T00:00:00.000Z"
         },
         "dimensionsSpec": {
           "dimensions": [],
           "dimensionExclusions": [
             "__time",
             "!!!_no_such_column_!!!"
           ],
           "includeAllDimensions": false,
           "useSchemaDiscovery": false
         },
         "metricsSpec": [],
         "granularitySpec": {
           "type": "uniform",
           "segmentGranularity": "DAY",
           "queryGranularity": {
             "type": "none"
           },
           "rollup": true,
           "intervals": []
         },
         "transformSpec": {
           "filter": null,
           "transforms": []
         }
       },
       "ioConfig": {
         "type": "index",
         "inputSource": {
           "type": "inline",
           "data": "{\"_index\":\"ebsn-log-search_2024_05_20\",\"_type\":\"_doc\",\"_id\":\"18e9340f-44b2-4c1a-9f9d-dacab3d32729\",\"_version\":1,\"_score\":1,\"_source\":{\"logSearchId\":301260,\"logType\":\"CAT\",\"term\":\"mesh\",\"searchedTerm\":\"mele\",\"searchedTermCorrectionType\":\"SUGG\",\"stemmedTerm\":null,\"refCategoryId\":null,\"refProductId\":null,\"advLayout\":null,\"results\":69,\"searchedTermResults\":0,\"page\":1,\"pageSize\":null,\"userId\":-1,\"ssoUserId\":null,\"addDate\":1716083010941,\"search\":true,\"filtered\":false,\"mainSearch\":true,\"mainWarehouseId\":140,\"params\":null,\"searchUid\":\"18e9340f-44b2-4c1a-9f9d-dacab3d32729\" }}"
         },
         "inputFormat": {
           "type": "json",
           "flattenSpec": {
             "useFieldDiscovery": true,
             "fields": [
               {
                 "type": "root",
                 "name": "_index",
                 "expr": "_index",
                 "nodes": null
               },
               {
                 "type": "root",
                 "name": "_type",
                 "expr": "_type",
                 "nodes": null
               },
               {
                 "type": "root",
                 "name": "_id",
                 "expr": "_id",
                 "nodes": null
               },
               {
                 "type": "root",
                 "name": "_version",
                 "expr": "_version",
                 "nodes": null
               },
               {
                 "type": "root",
                 "name": "_score",
                 "expr": "_score",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "logSearchId",
                 "expr": "$._source.logSearchId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "logType",
                 "expr": "$._source.logType",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "term",
                 "expr": "$._source.term",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "searchedTerm",
                 "expr": "$._source.searchedTerm",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "searchedTermCorrectionType",
                 "expr": "$._source.searchedTermCorrectionType",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "stemmedTerm",
                 "expr": "$._source.stemmedTerm",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "refCategoryId",
                 "expr": "$._source.refCategoryId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "refProductId",
                 "expr": "$._source.refProductId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "advLayout",
                 "expr": "$._source.advLayout",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "results",
                 "expr": "$._source.results",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "searchedTermResults",
                 "expr": "$._source.searchedTermResults",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "page",
                 "expr": "$._source.page",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "pageSize",
                 "expr": "$._source.pageSize",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "userId",
                 "expr": "$._source.userId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "ssoUserId",
                 "expr": "$._source.ssoUserId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "addDate",
                 "expr": "$._source.addDate",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "search",
                 "expr": "$._source.search",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "filtered",
                 "expr": "$._source.filtered",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "mainSearch",
                 "expr": "$._source.mainSearch",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "mainWarehouseId",
                 "expr": "$._source.mainWarehouseId",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "params",
                 "expr": "$._source.params",
                 "nodes": null
               },
               {
                 "type": "path",
                 "name": "searchUid",
                 "expr": "$._source.searchUid",
                 "nodes": null
               }
             ]
           },
           "disableNullColumnSkipping": true,
           "assumeNewlineDelimited": false,
           "useJsonNodeReader": false
         },
         "appendToExisting": false,
         "dropExisting": false
       },
       "tuningConfig": {
         "type": "index",
         "maxRowsPerSegment": null,
         "appendableIndexSpec": {
           "type": "onheap",
           "preserveExistingMetrics": false
         },
         "maxRowsInMemory": 1000000,
         "maxBytesInMemory": 0,
         "skipBytesInMemoryOverheadCheck": false,
         "maxTotalRows": null,
         "numShards": null,
         "partitionDimensions": [],
         "partitionsSpec": null,
         "indexSpec": {
           "bitmap": {
             "type": "roaring"
           },
           "dimensionCompression": "lz4",
           "stringDictionaryEncoding": {
             "type": "utf8"
           },
           "metricCompression": "lz4",
           "longEncoding": "longs"
         },
         "indexSpecForIntermediatePersists": {
           "bitmap": {
             "type": "roaring"
           },
           "dimensionCompression": "lz4",
           "stringDictionaryEncoding": {
             "type": "utf8"
           },
           "metricCompression": "lz4",
           "longEncoding": "longs"
         },
         "maxPendingPersists": 0,
         "forceGuaranteedRollup": false,
         "reportParseExceptions": false,
         "pushTimeout": 0,
         "segmentWriteOutMediumFactory": null,
         "logParseExceptions": false,
         "maxParseExceptions": 2147483647,
         "maxSavedParseExceptions": 0,
         "maxColumnsToMerge": -1,
         "awaitSegmentAvailabilityTimeoutMillis": 0
       }
     },
     "context": {
       "forceTimeChunkLock": true,
       "useLineageBasedSegmentAllocation": true
     },
     "groupId": "index_sample_kagaeeac_2024-05-23T13:23:44.758Z",
     "dataSource": "sample"
   }`
   
   but after the task execution I don't see the columns with null value I setted the attribute "disableNullColumnSkipping": true, although, someone can help me about this? thanks


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org