You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2023/05/12 04:09:24 UTC
[druid] branch 26.0.0 updated: fix segment metadata queries for auto ingested columns that had all null values (#14263)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch 26.0.0
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/26.0.0 by this push:
new 3542655df6 fix segment metadata queries for auto ingested columns that had all null values (#14263)
3542655df6 is described below
commit 3542655df6919c48c3e97d6af78f69025a3757e2
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Thu May 11 21:09:17 2023 -0700
fix segment metadata queries for auto ingested columns that had all null values (#14263)
---
.../druid/query/metadata/SegmentAnalyzer.java | 6 +++-
.../druid/segment/AutoTypeColumnIndexer.java | 4 +++
.../druid/query/metadata/SegmentAnalyzerTest.java | 35 ++++++++++++++++++++++
.../org/apache/druid/segment/IndexBuilder.java | 6 ++++
.../java/org/apache/druid/segment/TestHelper.java | 11 +++++++
5 files changed, 61 insertions(+), 1 deletion(-)
diff --git a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
index 7704a8e4c8..acee5b241b 100644
--- a/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
+++ b/processing/src/main/java/org/apache/druid/query/metadata/SegmentAnalyzer.java
@@ -344,7 +344,8 @@ public class SegmentAnalyzer
final TypeSignature<ValueType> typeSignature = capabilities == null ? ColumnType.UNKNOWN_COMPLEX : capabilities;
final String typeName = typeSignature.getComplexTypeName();
- try (final ComplexColumn complexColumn = columnHolder != null ? (ComplexColumn) columnHolder.getColumn() : null) {
+ try (final BaseColumn theColumn = columnHolder != null ? columnHolder.getColumn() : null) {
+ final ComplexColumn complexColumn = (ComplexColumn) theColumn;
final boolean hasMultipleValues = capabilities != null && capabilities.hasMultipleValues().isTrue();
final boolean hasNulls = capabilities != null && capabilities.hasNulls().isMaybeTrue();
long size = 0;
@@ -387,6 +388,9 @@ public class SegmentAnalyzer
null
);
}
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
private ColumnAnalysis analyzeArrayColumn(final ColumnCapabilities capabilities)
diff --git a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
index 38d8c51a3b..f541b33540 100644
--- a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
+++ b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnIndexer.java
@@ -282,6 +282,10 @@ public class AutoTypeColumnIndexer implements DimensionIndexer<StructuredData, S
private ColumnType getLogicalType()
{
+ if (fieldIndexers.isEmpty()) {
+ // we didn't see anything, so we can be anything, so why not a string?
+ return ColumnType.STRING;
+ }
if (fieldIndexers.size() == 1 && fieldIndexers.containsKey(NestedPathFinder.JSON_PATH_ROOT)) {
FieldIndexer rootField = fieldIndexers.get(NestedPathFinder.JSON_PATH_ROOT);
ColumnType singleType = rootField.getTypes().getSingleType();
diff --git a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
index be6466520d..adedb9c9e8 100644
--- a/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
+++ b/processing/src/test/java/org/apache/druid/query/metadata/SegmentAnalyzerTest.java
@@ -19,9 +19,13 @@
package org.apache.druid.query.metadata;
+import com.google.common.collect.ImmutableList;
import com.google.common.io.CharSource;
import com.google.common.io.Resources;
+import org.apache.druid.data.input.InputRowSchema;
import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.MapInputRowParser;
import org.apache.druid.data.input.impl.TimestampSpec;
import org.apache.druid.java.util.common.DateTimes;
import org.apache.druid.query.QueryPlus;
@@ -42,8 +46,11 @@ import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
import org.apache.druid.query.spec.LegacySegmentSpec;
import org.apache.druid.segment.ColumnSelectorFactory;
import org.apache.druid.segment.IncrementalIndexSegment;
+import org.apache.druid.segment.IndexBuilder;
+import org.apache.druid.segment.QueryableIndex;
import org.apache.druid.segment.QueryableIndexSegment;
import org.apache.druid.segment.Segment;
+import org.apache.druid.segment.TestHelper;
import org.apache.druid.segment.TestIndex;
import org.apache.druid.segment.column.ColumnBuilder;
import org.apache.druid.segment.column.ColumnHolder;
@@ -348,6 +355,34 @@ public class SegmentAnalyzerTest extends InitializedNullHandlingTest
}
}
+ @Test
+ public void testComplexAnalysisNullColumn() throws IOException
+ {
+ IndexBuilder bob = IndexBuilder.create();
+ bob.tmpDir(temporaryFolder.newFolder());
+ bob.writeNullColumns(true);
+ InputRowSchema schema = new InputRowSchema(
+ new TimestampSpec("time", null, null),
+ DimensionsSpec.builder().useSchemaDiscovery(true).build(),
+ null
+ );
+ bob.schema(IncrementalIndexSchema.builder()
+ .withTimestampSpec(schema.getTimestampSpec())
+ .withDimensionsSpec(schema.getDimensionsSpec())
+ .build());
+ bob.rows(ImmutableList.of(
+ MapInputRowParser.parse(schema, TestHelper.makeMapWithExplicitNull("time", 1234L, "x", null)))
+ );
+
+ QueryableIndex queryableIndex = bob.buildMMappedIndex();
+ Segment s = new QueryableIndexSegment(queryableIndex, SegmentId.dummy("test"));
+
+ SegmentAnalyzer analyzer = new SegmentAnalyzer(EMPTY_ANALYSES);
+ Map<String, ColumnAnalysis> analysis = analyzer.analyze(s);
+ Assert.assertEquals(ColumnType.STRING, analysis.get("x").getTypeSignature());
+ Assert.assertFalse(analysis.get("x").isError());
+ }
+
private static final class DummyObjectStrategy implements ObjectStrategy
{
diff --git a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
index 0ade7993d4..842e351fa5 100644
--- a/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
+++ b/processing/src/test/java/org/apache/druid/segment/IndexBuilder.java
@@ -129,6 +129,12 @@ public class IndexBuilder
return this;
}
+ public IndexBuilder writeNullColumns(boolean shouldWriteNullColumns)
+ {
+ this.indexMerger = new IndexMergerV9(jsonMapper, indexIO, segmentWriteOutMediumFactory, shouldWriteNullColumns);
+ return this;
+ }
+
public IndexBuilder indexSpec(IndexSpec indexSpec)
{
this.indexSpec = indexSpec;
diff --git a/processing/src/test/java/org/apache/druid/segment/TestHelper.java b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
index b0136094a6..b68aafa788 100644
--- a/processing/src/test/java/org/apache/druid/segment/TestHelper.java
+++ b/processing/src/test/java/org/apache/druid/segment/TestHelper.java
@@ -484,6 +484,17 @@ public class TestHelper
return theVals;
}
+ public static Map<String, Object> makeMapWithExplicitNull(Object... vals)
+ {
+ Preconditions.checkArgument(vals.length % 2 == 0);
+
+ Map<String, Object> theVals = new HashMap<>();
+ for (int i = 0; i < vals.length; i += 2) {
+ theVals.put(vals[i].toString(), vals[i + 1]);
+ }
+ return theVals;
+ }
+
public static void testSerializesDeserializes(Object object)
{
testSerializesDeserializes(JSON_MAPPER, object);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org