You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by "clintropolis (via GitHub)" <gi...@apache.org> on 2026/03/16 19:04:31 UTC
[PR] remove deprecated InputRowParser from SeekableStreamIndexTaskRunner (kafka, kinesis, etc) (druid)
clintropolis opened a new pull request, #19166:
URL: https://github.com/apache/druid/pull/19166
### Description
This has been deprecated forever, after #19111 all of the streaming data formats now have `InputFormat` implementations so this can finally be removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
Re: [PR] remove deprecated InputRowParser from SeekableStreamIndexTaskRunner (kafka, kinesis, etc) (druid)
Posted by "clintropolis (via GitHub)" <gi...@apache.org>.
clintropolis merged PR #19166:
URL: https://github.com/apache/druid/pull/19166
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
Re: [PR] remove deprecated InputRowParser from SeekableStreamIndexTaskRunner (kafka, kinesis, etc) (druid)
Posted by "github-advanced-security[bot] (via GitHub)" <gi...@apache.org>.
github-advanced-security[bot] commented on code in PR #19166:
URL: https://github.com/apache/druid/pull/19166#discussion_r2942487505
##########
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroStreamInputFormatTest.java:
##########
@@ -106,6 +119,55 @@
private static final String NESTED_ARRAY_VAL = "nestedArrayVal";
private static final String IS_VALID = "isValid";
private static final String TOPIC = "aTopic";
+ private static final String EVENT_TYPE_VALUE = "type-a";
+ private static final long ID_VALUE = 1976491L;
+ private static final long SOME_OTHER_ID_VALUE = 6568719896L;
+ private static final float SOME_FLOAT_VALUE = 0.23555f;
+ private static final int SOME_INT_VALUE = 1;
+ private static final long SOME_LONG_VALUE = 679865987569912369L;
+ private static final ZonedDateTime DATE_TIME = ZonedDateTime.of(2015, 10, 25, 19, 30, 0, 0, ZoneOffset.UTC);
+ private static final MyFixed SOME_FIXED_VALUE = new MyFixed(ByteBuffer.allocate(16).array());
+ private static final long SUB_LONG_VALUE = 1543698L;
+ private static final int SUB_INT_VALUE = 4892;
+ private static final MySubRecord SOME_RECORD_VALUE = MySubRecord.newBuilder()
+ .setSubInt(SUB_INT_VALUE)
+ .setSubLong(SUB_LONG_VALUE)
+ .build();
+ private static final List<CharSequence> SOME_STRING_ARRAY_VALUE = Arrays.asList("8", "4", "2", "1", null);
+ private static final List<Integer> SOME_INT_ARRAY_VALUE = Arrays.asList(1, 2, 4, 8);
+ static final Map<CharSequence, Integer> SOME_INT_VALUE_MAP_VALUE = Maps.asMap(
+ new HashSet<>(Arrays.asList("8", "2", "4", "1")), new Function<>()
+ {
+ @Nonnull
+ @Override
+ public Integer apply(@Nullable CharSequence input)
+ {
+ return Integer.parseInt(input.toString());
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more details](https://github.com/apache/druid/security/code-scanning/10892)
##########
extensions-core/avro-extensions/src/test/java/org/apache/druid/data/input/AvroStreamInputFormatTest.java:
##########
@@ -434,4 +496,167 @@
assertInputRowCorrect(inputRow, DIMENSIONS_SCHEMALESS, false);
}
}
+
+ static void assertInputRowCorrect(InputRow inputRow, List<String> expectedDimensions, boolean isFromPigAvro)
+ {
+ Assert.assertEquals(expectedDimensions, inputRow.getDimensions());
+ Assert.assertEquals(1543698L, inputRow.getTimestampFromEpoch());
+
+ // test dimensions
+ Assert.assertEquals(Collections.singletonList(EVENT_TYPE_VALUE), inputRow.getDimension(EVENT_TYPE));
+ Assert.assertEquals(Collections.singletonList(String.valueOf(ID_VALUE)), inputRow.getDimension(ID));
+ Assert.assertEquals(
+ Collections.singletonList(String.valueOf(SOME_OTHER_ID_VALUE)),
+ inputRow.getDimension(SOME_OTHER_ID)
+ );
+ Assert.assertEquals(Collections.singletonList(String.valueOf(true)), inputRow.getDimension(IS_VALID));
+
+ // someRecordArray represents a record generated from Pig using AvroStorage
+ // as it implicitly converts array elements to a record
+ if (isFromPigAvro) {
+ Assert.assertEquals(
+ Collections.singletonList(SOME_RECORD_ARRAY_VALUE.get(0).getNestedString()),
+ inputRow.getDimension("someRecordArray")
+ );
+ } else {
+ Assert.assertEquals(
+ Lists.transform(SOME_INT_ARRAY_VALUE, String::valueOf),
+ inputRow.getDimension("someIntArray")
+ );
+ // For string array, nulls are preserved so use ArrayList (ImmutableList doesn't support nulls)
+ Assert.assertEquals(
+ SOME_STRING_ARRAY_VALUE.stream().map(Evals::asString).collect(Collectors.toList()),
+ inputRow.getDimension("someStringArray")
+ );
+
+ final Object someRecordArrayObj = inputRow.getRaw("someRecordArray");
+ Assert.assertNotNull(someRecordArrayObj);
+ Assert.assertTrue(someRecordArrayObj instanceof List);
+ Assert.assertEquals(1, ((List) someRecordArrayObj).size());
+ final Object recordArrayElementObj = ((List) someRecordArrayObj).get(0);
+ Assert.assertNotNull(recordArrayElementObj);
+ Assert.assertTrue(recordArrayElementObj instanceof LinkedHashMap);
+ LinkedHashMap recordArrayElement = (LinkedHashMap) recordArrayElementObj;
+ Assert.assertEquals("string in record", recordArrayElement.get("nestedString"));
+ }
+
+ final Object someIntValueMapObj = inputRow.getRaw("someIntValueMap");
+ Assert.assertNotNull(someIntValueMapObj);
+ Assert.assertTrue(someIntValueMapObj instanceof LinkedHashMap);
+ LinkedHashMap someIntValueMap = (LinkedHashMap) someIntValueMapObj;
+ Assert.assertEquals(4, someIntValueMap.size());
+ Assert.assertEquals(1, someIntValueMap.get("1"));
+ Assert.assertEquals(2, someIntValueMap.get("2"));
+ Assert.assertEquals(4, someIntValueMap.get("4"));
+ Assert.assertEquals(8, someIntValueMap.get("8"));
+
+
+ final Object someStringValueMapObj = inputRow.getRaw("someStringValueMap");
+ Assert.assertNotNull(someStringValueMapObj);
+ Assert.assertTrue(someStringValueMapObj instanceof LinkedHashMap);
+ LinkedHashMap someStringValueMap = (LinkedHashMap) someStringValueMapObj;
+ Assert.assertEquals(4, someStringValueMap.size());
+ Assert.assertEquals("1", someStringValueMap.get("1"));
+ Assert.assertEquals("2", someStringValueMap.get("2"));
+ Assert.assertEquals("4", someStringValueMap.get("4"));
+ Assert.assertEquals("8", someStringValueMap.get("8"));
+
+
+ final Object someRecordObj = inputRow.getRaw("someRecord");
+ Assert.assertNotNull(someRecordObj);
+ Assert.assertTrue(someRecordObj instanceof LinkedHashMap);
+ LinkedHashMap someRecord = (LinkedHashMap) someRecordObj;
+ Assert.assertEquals(4892, someRecord.get("subInt"));
+ Assert.assertEquals(1543698L, someRecord.get("subLong"));
+
+ final Object someList = inputRow.getDimension("nestedArrayVal");
+ Assert.assertNotNull(someList);
+ Assert.assertTrue(someList instanceof List);
+ List someRecordObj3List = (List) someList;
+ Assert.assertEquals(1, someRecordObj3List.size());
+ Assert.assertEquals("string in record", someRecordObj3List.get(0));
+
+
+ // towards Map avro field as druid dimension, need to convert its toString() back to HashMap to check equality
+ Assert.assertEquals(1, inputRow.getDimension("someIntValueMap").size());
+ Assert.assertEquals(
+ SOME_INT_VALUE_MAP_VALUE,
+ new HashMap<CharSequence, Integer>(
+ Maps.transformValues(
+ Splitter
+ .on(",")
+ .withKeyValueSeparator("=")
+ .split(BRACES_AND_SPACE.matcher(inputRow.getDimension("someIntValueMap").get(0)).replaceAll("")),
+ new Function<>()
+ {
+ @Nullable
+ @Override
+ public Integer apply(@Nullable String input)
+ {
+ return Integer.valueOf(input);
Review Comment:
## Missing catch of NumberFormatException
Potential uncaught 'java.lang.NumberFormatException'.
[Show more details](https://github.com/apache/druid/security/code-scanning/10893)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org