You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2026/07/28 22:44:59 UTC
(druid) branch master updated: feat: add support for specifying DimensionSchema to catalog clustered segment spec (#19770)
This is an automated email from the ASF dual-hosted git repository.
clintropolis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new ea690003d09 feat: add support for specifying DimensionSchema to catalog clustered segment spec (#19770)
ea690003d09 is described below
commit ea690003d09f5f7ef4376bd2b65ab07da2c2f822
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Tue Jul 28 15:44:52 2026 -0700
feat: add support for specifying DimensionSchema to catalog clustered segment spec (#19770)
---
.../catalog/CatalogIngestAndQueryTest.java | 2 +-
.../org/apache/druid/msq/exec/MSQInsertTest.java | 9 +-
.../ClusteredValueGroupsBaseTableMetadata.java | 136 +++++++++++-
.../ClusteredValueGroupsBaseTableMetadataTest.java | 234 ++++++++++++++++++++-
.../catalog/model/table/DatasourceTableTest.java | 13 +-
.../calcite/CalciteCatalogIngestionDmlTest.java | 7 +
6 files changed, 384 insertions(+), 17 deletions(-)
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogIngestAndQueryTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogIngestAndQueryTest.java
index 1dcca2c4a8f..d1078439d73 100644
--- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogIngestAndQueryTest.java
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/catalog/CatalogIngestAndQueryTest.java
@@ -460,7 +460,7 @@ public abstract class CatalogIngestAndQueryTest extends CatalogTestBase
.sealed(true)
.property(
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(ImmutableList.of("varchar_col2"), null)
+ new ClusteredValueGroupsBaseTableMetadata(ImmutableList.of("varchar_col2"), null, null)
)
.property(
DatasourceDefn.PROJECTIONS_KEYS_PROPERTY,
diff --git a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java
index 8e00d2780b9..f425fdfbd0e 100644
--- a/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java
+++ b/multi-stage-query/src/test/java/org/apache/druid/msq/exec/MSQInsertTest.java
@@ -69,6 +69,7 @@ import org.apache.druid.query.groupby.orderby.DefaultLimitSpec;
import org.apache.druid.query.groupby.orderby.OrderByColumnSpec;
import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
import org.apache.druid.segment.AggregateProjectionMetadata;
+import org.apache.druid.segment.AutoTypeColumnSchema;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.segment.column.RowSignature;
import org.apache.druid.segment.column.ValueType;
@@ -248,7 +249,13 @@ public class MSQInsertTest extends MSQTestBase
.column("delta", Columns.SQL_BIGINT)
.sealed(true)
.baseTable(
- new ClusteredValueGroupsBaseTableMetadata(ImmutableList.of("channel"), null)
+ new ClusteredValueGroupsBaseTableMetadata(
+ ImmutableList.of("channel"),
+ null,
+ // customize the segment-creation schema of a declared column: 'user' is stored as an
+ // auto column cast to its declared VARCHAR type (the logical schema is unchanged)
+ ImmutableList.of(new AutoTypeColumnSchema("user", ColumnType.STRING, null))
+ )
)
.property(
DatasourceDefn.PROJECTIONS_KEYS_PROPERTY,
diff --git a/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java b/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
index e84cc853c63..f0a3321285d 100644
--- a/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
+++ b/server/src/main/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadata.java
@@ -26,16 +26,20 @@ import com.fasterxml.jackson.annotation.JsonTypeName;
import org.apache.druid.data.input.impl.ClusteredValueGroupsBaseTableProjectionSpec;
import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.error.InvalidInput;
+import org.apache.druid.segment.AutoTypeColumnSchema;
import org.apache.druid.segment.NestedDataColumnSchema;
import org.apache.druid.segment.VirtualColumns;
+import org.apache.druid.segment.column.ColumnHolder;
import org.apache.druid.segment.column.ColumnType;
import org.apache.druid.utils.CollectionUtils;
import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -46,6 +50,10 @@ import java.util.Set;
* (column names, types, and order) is taken from the catalog column list by {@link #createSpec(List)}. The declared
* column order is the physical segment order, so, mirroring the physical spec, the clustering columns must be
* declared as the leading prefix of the column list.
+ * <p>
+ * The optional {@link #columnSchemas} do NOT define columns, the catalog column list remains the logical schema,
+ * they are per-column customizations of the exact {@link DimensionSchema} used to create segments for a declared
+ * column, replacing the default schema derived from the declared column type.
*/
@JsonTypeName(ClusteredValueGroupsBaseTableMetadata.TYPE_NAME)
public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTableMetadata
@@ -54,15 +62,18 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
private final List<String> clusteringColumns;
private final VirtualColumns virtualColumns;
+ private final List<DimensionSchema> columnSchemas;
@JsonCreator
public ClusteredValueGroupsBaseTableMetadata(
@JsonProperty("clusteringColumns") List<String> clusteringColumns,
- @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns
+ @JsonProperty("virtualColumns") @Nullable VirtualColumns virtualColumns,
+ @JsonProperty("columnSchemas") @Nullable List<DimensionSchema> columnSchemas
)
{
this.clusteringColumns = clusteringColumns == null ? Collections.emptyList() : clusteringColumns;
this.virtualColumns = virtualColumns == null ? VirtualColumns.EMPTY : virtualColumns;
+ this.columnSchemas = columnSchemas == null ? Collections.emptyList() : columnSchemas;
}
@Override
@@ -86,12 +97,30 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
return virtualColumns;
}
+ /**
+ * Per-column customizations of the {@link DimensionSchema} used during segment creation, keyed by
+ * {@link DimensionSchema#getName()}; empty when every declared column uses the schema derived from its declared
+ * type. These do not define columns: every entry must customize a declared, non-clustering column.
+ */
+ @JsonProperty("columnSchemas")
+ @JsonInclude(JsonInclude.Include.NON_EMPTY)
+ public List<DimensionSchema> getColumnSchemas()
+ {
+ return columnSchemas;
+ }
+
/**
* Creates the physical spec from the declared catalog columns, used verbatim: the declared column order is the
* physical segment order, so the clustering columns must be declared as the leading prefix of the column list (in
* {@link #clusteringColumns} order); anything else is a validation error. Every clustering column must be a declared
* column, a clustering column computed by a virtual column at ingest time is still a stored, queryable column, so it
* too must appear in the column list. All ordering and layout rules are enforced by the spec itself.
+ * <p>
+ * A column with an entry in {@link #columnSchemas} uses that {@link DimensionSchema} verbatim in place of the
+ * default derived from its declared type. Customizations may only target declared, non-clustering columns (a
+ * clustering column's physical representation is fixed by the clustered format, and {@code __time} is always a
+ * long), and the schema's type must match the declared logical type so the physical schema cannot silently
+ * contradict the SQL schema.
*/
@Override
public ClusteredValueGroupsBaseTableProjectionSpec createSpec(List<ColumnSpec> columns)
@@ -102,11 +131,12 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
TYPE_NAME
);
}
+ final Map<String, DimensionSchema> customSchemas = indexColumnSchemas();
final Set<String> declaredNames = new HashSet<>();
final List<DimensionSchema> specColumns = new ArrayList<>(columns.size());
for (ColumnSpec column : columns) {
declaredNames.add(column.name());
- specColumns.add(toDimensionSchema(column));
+ specColumns.add(toDimensionSchema(column, customSchemas.get(column.name())));
}
for (String clusteringColumn : clusteringColumns) {
if (!declaredNames.contains(clusteringColumn)) {
@@ -118,6 +148,16 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
);
}
}
+ for (String customized : customSchemas.keySet()) {
+ if (!declaredNames.contains(customized)) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] does not customize a declared column; column schemas do not define columns,"
+ + " declare [%s] in the table's column list",
+ customized,
+ customized
+ );
+ }
+ }
return ClusteredValueGroupsBaseTableProjectionSpec.builder()
.virtualColumns(virtualColumns)
.columns(specColumns)
@@ -125,12 +165,30 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
.build();
}
- private static DimensionSchema toDimensionSchema(ColumnSpec column)
+ private Map<String, DimensionSchema> indexColumnSchemas()
+ {
+ final Map<String, DimensionSchema> customSchemas = new HashMap<>();
+ for (DimensionSchema schema : columnSchemas) {
+ if (schema == null) {
+ throw InvalidInput.exception("columnSchemas must not contain null entries");
+ }
+ if (customSchemas.put(schema.getName(), schema) != null) {
+ throw InvalidInput.exception("columnSchemas contains duplicate entries for column [%s]", schema.getName());
+ }
+ }
+ return customSchemas;
+ }
+
+ private DimensionSchema toDimensionSchema(ColumnSpec column, @Nullable DimensionSchema customSchema)
{
ColumnType druidType = Columns.druidType(column);
if (druidType == null) {
druidType = ColumnType.STRING;
}
+ if (customSchema != null) {
+ validateColumnSchemaCustomization(column, customSchema, druidType);
+ return customSchema;
+ }
if (druidType.isPrimitive() || druidType.isPrimitiveArray()) {
// The declared type is retained in the ingestion schema (primitive arrays are cast, rather than left to an
// untyped auto column whose type is inferred from the ingested values; note that the auto schema stores
@@ -150,6 +208,72 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
);
}
+ private void validateColumnSchemaCustomization(
+ ColumnSpec column,
+ DimensionSchema customSchema,
+ ColumnType declaredType
+ )
+ {
+ if (ColumnHolder.TIME_COLUMN_NAME.equals(column.name())) {
+ throw InvalidInput.exception(
+ "columnSchemas cannot customize [%s]: the time column is always stored as a long",
+ ColumnHolder.TIME_COLUMN_NAME
+ );
+ }
+ if (clusteringColumns.contains(column.name())) {
+ throw InvalidInput.exception(
+ "columnSchemas cannot customize clustering column [%s]: the physical representation of clustering columns"
+ + " is fixed by the clustered segment format",
+ column.name()
+ );
+ }
+ // The schema's type must match the declared logical type, so the physical schema cannot silently contradict the
+ // SQL schema that INSERT/REPLACE queries are validated and coerced against.
+ ColumnType expectedType = declaredType;
+ if (customSchema instanceof AutoTypeColumnSchema) {
+ // An uncast auto column stores values as they are ingested (inferring the physical type) rather than coercing
+ // them to the declared type; only a column declared COMPLEX<json> may store arbitrary shapes.
+ if (((AutoTypeColumnSchema) customSchema).getCastToType() == null
+ && !ColumnType.NESTED_DATA.equals(declaredType)) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] is an auto column schema without a castToType; an uncast auto column stores"
+ + " values as they are ingested rather than coercing them to the column's declared type [%s], set"
+ + " castToType to match the declared type",
+ column.name(),
+ declaredType
+ );
+ }
+ // The auto schema stores FLOAT as DOUBLE.
+ expectedType = autoColumnType(declaredType);
+ }
+ // A NestedDataColumnSchema always reports COMPLEX<json>, so this also restricts json schemas to columns declared
+ // as COMPLEX<json>.
+ if (!expectedType.equals(customSchema.getColumnType())) {
+ throw InvalidInput.exception(
+ "columnSchemas entry [%s] of type [%s] does not match the column's declared type [%s]; column schemas"
+ + " customize the physical representation of a declared column, not its type",
+ column.name(),
+ customSchema.getColumnType(),
+ declaredType
+ );
+ }
+ }
+
+ /**
+ * The type the auto schema stores for a declared type: {@link AutoTypeColumnSchema} coerces FLOAT to DOUBLE (the
+ * default derivation for declared FLOAT ARRAY columns relies on the same coercion).
+ */
+ private static ColumnType autoColumnType(ColumnType declaredType)
+ {
+ if (ColumnType.FLOAT.equals(declaredType)) {
+ return ColumnType.DOUBLE;
+ }
+ if (ColumnType.FLOAT_ARRAY.equals(declaredType)) {
+ return ColumnType.DOUBLE_ARRAY;
+ }
+ return declaredType;
+ }
+
@Override
public boolean equals(Object o)
{
@@ -158,13 +282,14 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
}
ClusteredValueGroupsBaseTableMetadata that = (ClusteredValueGroupsBaseTableMetadata) o;
return Objects.equals(clusteringColumns, that.clusteringColumns)
- && Objects.equals(virtualColumns, that.virtualColumns);
+ && Objects.equals(virtualColumns, that.virtualColumns)
+ && Objects.equals(columnSchemas, that.columnSchemas);
}
@Override
public int hashCode()
{
- return Objects.hash(clusteringColumns, virtualColumns);
+ return Objects.hash(clusteringColumns, virtualColumns, columnSchemas);
}
@Override
@@ -173,6 +298,7 @@ public class ClusteredValueGroupsBaseTableMetadata implements DatasourceBaseTabl
return "ClusteredValueGroupsBaseTableMetadata{" +
"clusteringColumns=" + clusteringColumns +
", virtualColumns=" + virtualColumns +
+ ", columnSchemas=" + columnSchemas +
'}';
}
}
diff --git a/server/src/test/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadataTest.java b/server/src/test/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadataTest.java
index af3a956970a..e7891dd9c2a 100644
--- a/server/src/test/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadataTest.java
+++ b/server/src/test/java/org/apache/druid/catalog/model/ClusteredValueGroupsBaseTableMetadataTest.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.InjectableValues;
import com.fasterxml.jackson.databind.ObjectMapper;
import nl.jqno.equalsverifier.EqualsVerifier;
import org.apache.druid.data.input.impl.ClusteredValueGroupsBaseTableProjectionSpec;
+import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.data.input.impl.DoubleDimensionSchema;
import org.apache.druid.data.input.impl.LongDimensionSchema;
import org.apache.druid.data.input.impl.StringDimensionSchema;
@@ -31,6 +32,7 @@ import org.apache.druid.error.DruidException;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.math.expr.ExprMacroTable;
import org.apache.druid.segment.AutoTypeColumnSchema;
+import org.apache.druid.segment.DefaultColumnFormatConfig;
import org.apache.druid.segment.NestedDataColumnSchema;
import org.apache.druid.segment.VirtualColumns;
import org.apache.druid.segment.column.ColumnType;
@@ -47,7 +49,9 @@ import java.util.Map;
public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHandlingTest
{
private final ObjectMapper mapper = new DefaultObjectMapper().setInjectableValues(
- new InjectableValues.Std().addValue(ExprMacroTable.class, ExprMacroTable.nil())
+ new InjectableValues.Std()
+ .addValue(ExprMacroTable.class, ExprMacroTable.nil())
+ .addValue(DefaultColumnFormatConfig.class, new DefaultColumnFormatConfig(null, null, null, null))
);
// Declared order is the physical segment order: clustering columns lead, __time is an explicit positional column.
@@ -66,7 +70,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
Collections.singletonList("tenant_lower"),
VirtualColumns.create(
new ExpressionVirtualColumn("tenant_lower", "lower(\"tenant\")", ColumnType.STRING, ExprMacroTable.nil())
- )
+ ), null
);
final String json = mapper.writeValueAsString(metadata);
final DatasourceBaseTableMetadata fromJson = mapper.readValue(json, DatasourceBaseTableMetadata.class);
@@ -78,10 +82,29 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Arrays.asList("tenant", "region"),
+ null,
null
);
final String json = mapper.writeValueAsString(metadata);
Assert.assertFalse(json.contains("virtualColumns"));
+ Assert.assertFalse(json.contains("columnSchemas"));
+ final DatasourceBaseTableMetadata fromJson = mapper.readValue(json, DatasourceBaseTableMetadata.class);
+ Assert.assertEquals(metadata, fromJson);
+ }
+
+ @Test
+ public void testSerdeWithColumnSchemas() throws Exception
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Arrays.asList(
+ new StringDimensionSchema("region", DimensionSchema.MultiValueHandling.ARRAY, false),
+ new AutoTypeColumnSchema("value", ColumnType.DOUBLE, null),
+ new NestedDataColumnSchema("attrs", NestedDataColumnSchema.DEFAULT_FORMAT_VERSION)
+ )
+ );
+ final String json = mapper.writeValueAsString(metadata);
final DatasourceBaseTableMetadata fromJson = mapper.readValue(json, DatasourceBaseTableMetadata.class);
Assert.assertEquals(metadata, fromJson);
}
@@ -93,6 +116,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
// the type discriminator must survive that path (it is an EXISTING_PROPERTY for this reason).
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
final String json = mapper.writeValueAsString(Collections.singletonMap("baseTable", metadata));
@@ -109,6 +133,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
// The declared column order is the physical segment order, used verbatim; types map through Columns.druidType
@@ -139,7 +164,8 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
);
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant_lower"),
- virtualColumns
+ virtualColumns,
+ null
);
final List<ColumnSpec> columns = Arrays.asList(
new ColumnSpec("tenant_lower", Columns.SQL_VARCHAR, null),
@@ -162,11 +188,203 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
);
}
+ @Test
+ public void testCreateSpecWithColumnSchemas()
+ {
+ // Column schemas customize the exact DimensionSchema used at segment creation without defining columns: the
+ // declared column list remains the logical schema. 'region' keeps its declared STRING type but customizes
+ // multi-value handling and disables the bitmap index; 'value' is stored as an auto column cast to its declared
+ // DOUBLE type.
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Arrays.asList(
+ new StringDimensionSchema("region", DimensionSchema.MultiValueHandling.ARRAY, false),
+ new AutoTypeColumnSchema("value", ColumnType.DOUBLE, null)
+ )
+ );
+ Assert.assertEquals(
+ ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .columns(
+ new StringDimensionSchema("tenant"),
+ new LongDimensionSchema(Columns.TIME_COLUMN),
+ new StringDimensionSchema("region", DimensionSchema.MultiValueHandling.ARRAY, false),
+ new LongDimensionSchema("delta"),
+ new AutoTypeColumnSchema("value", ColumnType.DOUBLE, null)
+ )
+ .clusteringColumns("tenant")
+ .build(),
+ metadata.createSpec(COLUMNS)
+ );
+ }
+
+ @Test
+ public void testCreateSpecAutoColumnSchemaCoercions()
+ {
+ // A declared FLOAT column may be customized with an auto schema cast to FLOAT: the auto schema itself stores
+ // FLOAT as DOUBLE (the same coercion the default derivation relies on for FLOAT arrays). A column declared
+ // COMPLEX<json> may use an uncast auto schema, since arbitrary shapes are that type's contract.
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Arrays.asList(
+ new AutoTypeColumnSchema("ratio", ColumnType.FLOAT, null),
+ AutoTypeColumnSchema.of("attrs")
+ )
+ );
+ final List<ColumnSpec> columns = Arrays.asList(
+ new ColumnSpec("tenant", Columns.SQL_VARCHAR, null),
+ new ColumnSpec(Columns.TIME_COLUMN, null, null),
+ new ColumnSpec("ratio", Columns.SQL_FLOAT, null),
+ new ColumnSpec("attrs", ColumnType.NESTED_DATA.asTypeString(), null)
+ );
+ Assert.assertEquals(
+ ClusteredValueGroupsBaseTableProjectionSpec.builder()
+ .columns(
+ new StringDimensionSchema("tenant"),
+ new LongDimensionSchema(Columns.TIME_COLUMN),
+ new AutoTypeColumnSchema("ratio", ColumnType.FLOAT, null),
+ AutoTypeColumnSchema.of("attrs")
+ )
+ .clusteringColumns("tenant")
+ .build(),
+ metadata.createSpec(columns)
+ );
+ }
+
+ @Test
+ public void testCreateSpecUncastAutoColumnSchemaFails()
+ {
+ // An uncast auto column stores values as ingested rather than coercing to the declared type, so it is only
+ // legal for columns declared COMPLEX<json>.
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(AutoTypeColumnSchema.of("value"))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(
+ e.getMessage().contains("columnSchemas entry [value] is an auto column schema without a castToType")
+ );
+ }
+
+ @Test
+ public void testCreateSpecAutoColumnSchemaCastMismatchFails()
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new AutoTypeColumnSchema("region", ColumnType.LONG, null))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(
+ e.getMessage().contains("columnSchemas entry [region] of type [LONG] does not match the column's declared type [STRING]")
+ );
+ }
+
+ @Test
+ public void testCreateSpecJsonColumnSchemaForNonJsonColumnFails()
+ {
+ // A json schema always stores COMPLEX<json>, so it may only customize columns declared as COMPLEX<json>.
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new NestedDataColumnSchema("region", NestedDataColumnSchema.DEFAULT_FORMAT_VERSION))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(
+ e.getMessage().contains(
+ "columnSchemas entry [region] of type [COMPLEX<json>] does not match the column's declared type [STRING]"
+ )
+ );
+ }
+
+ @Test
+ public void testCreateSpecColumnSchemaForUndeclaredColumnFails()
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new StringDimensionSchema("no_such_column"))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(
+ e.getMessage().contains("columnSchemas entry [no_such_column] does not customize a declared column")
+ );
+ }
+
+ @Test
+ public void testCreateSpecColumnSchemaForClusteringColumnFails()
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new StringDimensionSchema("tenant", DimensionSchema.MultiValueHandling.ARRAY, false))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(e.getMessage().contains("columnSchemas cannot customize clustering column [tenant]"));
+ }
+
+ @Test
+ public void testCreateSpecColumnSchemaForTimeColumnFails()
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new LongDimensionSchema(Columns.TIME_COLUMN))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(e.getMessage().contains("columnSchemas cannot customize [__time]"));
+ }
+
+ @Test
+ public void testCreateSpecColumnSchemaTypeMismatchFails()
+ {
+ // 'region' is declared as STRING; a LONG dimension schema would contradict the logical schema SQL writes are
+ // validated against.
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Collections.singletonList(new LongDimensionSchema("region"))
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(
+ e.getMessage().contains("columnSchemas entry [region] of type [LONG] does not match the column's declared type [STRING]")
+ );
+ }
+
+ @Test
+ public void testCreateSpecNullColumnSchemaEntryFails() throws Exception
+ {
+ final DatasourceBaseTableMetadata metadata = mapper.readValue(
+ "{\"type\":\"clusteredValueGroups\",\"clusteringColumns\":[\"tenant\"],\"columnSchemas\":[null]}",
+ DatasourceBaseTableMetadata.class
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(e.getMessage().contains("columnSchemas must not contain null entries"));
+ }
+
+ @Test
+ public void testCreateSpecDuplicateColumnSchemasFails()
+ {
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
+ Collections.singletonList("tenant"),
+ null,
+ Arrays.asList(
+ new StringDimensionSchema("region"),
+ new StringDimensionSchema("region", DimensionSchema.MultiValueHandling.ARRAY, false)
+ )
+ );
+ final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
+ Assert.assertTrue(e.getMessage().contains("columnSchemas contains duplicate entries for column [region]"));
+ }
+
@Test
public void testCreateSpecMultipleClusteringColumns()
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Arrays.asList("region", "tenant"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -194,6 +412,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -228,6 +447,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -246,6 +466,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
// physical segment order, so this is an error rather than a silent reorder.
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("region"),
+ null,
null
);
final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
@@ -257,6 +478,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Arrays.asList("tenant", "region"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -273,6 +495,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("no_such_column"),
+ null,
null
);
final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
@@ -284,6 +507,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -299,6 +523,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tags"),
+ null,
null
);
final List<ColumnSpec> columns = Arrays.asList(
@@ -312,7 +537,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
@Test
public void testCreateSpecEmptyClusteringColumnsFails()
{
- final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(null, null);
+ final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(null, null, null);
final DruidException e = Assert.assertThrows(DruidException.class, () -> metadata.createSpec(COLUMNS));
Assert.assertTrue(e.getMessage().contains("clusteringColumns must be non-empty"));
}
@@ -322,6 +547,7 @@ public class ClusteredValueGroupsBaseTableMetadataTest extends InitializedNullHa
{
final DatasourceBaseTableMetadata metadata = new ClusteredValueGroupsBaseTableMetadata(
Collections.singletonList("tenant"),
+ null,
null
);
final DruidException e = Assert.assertThrows(
diff --git a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java
index b4e06cb1a01..287e9477c89 100644
--- a/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java
+++ b/server/src/test/java/org/apache/druid/catalog/model/table/DatasourceTableTest.java
@@ -162,7 +162,7 @@ public class DatasourceTableTest
ImmutableMap.of(
DatasourceDefn.SEALED_PROPERTY, true,
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null)
+ new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null, null)
),
columns
);
@@ -176,7 +176,7 @@ public class DatasourceTableTest
DatasourceDefn.TABLE_TYPE,
ImmutableMap.of(
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null)
+ new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null, null)
),
columns
);
@@ -193,7 +193,7 @@ public class DatasourceTableTest
ImmutableMap.of(
DatasourceDefn.SEALED_PROPERTY, true,
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("no_such_column"), null)
+ new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("no_such_column"), null, null)
),
columns
);
@@ -209,7 +209,7 @@ public class DatasourceTableTest
ImmutableMap.of(
DatasourceDefn.SEALED_PROPERTY, true,
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("region"), null)
+ new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("region"), null, null)
),
columns
);
@@ -224,7 +224,7 @@ public class DatasourceTableTest
ImmutableMap.of(
DatasourceDefn.SEALED_PROPERTY, true,
DatasourceDefn.BASE_TABLE_PROPERTY,
- new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null)
+ new ClusteredValueGroupsBaseTableMetadata(Collections.singletonList("tenant"), null, null)
),
Collections.singletonList(new ColumnSpec("tenant", Columns.SQL_VARCHAR, null))
);
@@ -244,7 +244,8 @@ public class DatasourceTableTest
Collections.singletonList("tenant_lower"),
VirtualColumns.create(
new ExpressionVirtualColumn("tenant_lower", "lower(\"tenant\")", ColumnType.STRING, ExprMacroTable.nil())
- )
+ ),
+ null
);
TableSpec spec = new TableSpec(
DatasourceDefn.TABLE_TYPE,
diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
index fcd498450c3..02958704557 100644
--- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
+++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCatalogIngestionDmlTest.java
@@ -33,7 +33,9 @@ import org.apache.druid.catalog.model.facade.DatasourceFacade;
import org.apache.druid.catalog.model.table.ClusterKeySpec;
import org.apache.druid.catalog.model.table.DatasourceDefn;
import org.apache.druid.data.input.impl.CsvInputFormat;
+import org.apache.druid.data.input.impl.DimensionSchema;
import org.apache.druid.data.input.impl.InlineInputSource;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
import org.apache.druid.error.DruidException;
import org.apache.druid.jackson.DefaultObjectMapper;
import org.apache.druid.java.util.common.StringUtils;
@@ -402,6 +404,11 @@ public abstract class CalciteCatalogIngestionDmlTest extends CalciteIngestionDml
ColumnType.STRING,
ExprMacroTable.nil()
)
+ ),
+ // customizes the segment-creation schema of 'dim1'; invisible to the planner, since
+ // the declared column list remains the logical schema
+ ImmutableList.of(
+ new StringDimensionSchema("dim1", DimensionSchema.MultiValueHandling.ARRAY, false)
)
)
),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org