You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2021/11/11 13:04:48 UTC
[druid] branch master updated: Add dimension partitioningType to
metrics to track usage of different partitioning schemes (#11902)
This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 223c569 Add dimension partitioningType to metrics to track usage of different partitioning schemes (#11902)
223c569 is described below
commit 223c5692a8292a210a770dc0702f75b4484ae347
Author: Kashif Faraz <ka...@gmail.com>
AuthorDate: Thu Nov 11 18:34:27 2021 +0530
Add dimension partitioningType to metrics to track usage of different partitioning schemes (#11902)
Add method ShardSpec.getType() to get name of shard spec type
List all names of shard spec types in the interface ShardSpec itself
for easy reference and maintenance
Add dimension partitioningType to metric segment/added/bytes
---
.../partition/BuildingDimensionRangeShardSpec.java | 8 ++-
.../BuildingHashBasedNumberedShardSpec.java | 8 ++-
.../partition/BuildingNumberedShardSpec.java | 7 ++-
.../BuildingSingleDimensionShardSpec.java | 8 ++-
.../partition/DimensionRangeBucketShardSpec.java | 8 ++-
.../partition/DimensionRangeShardSpec.java | 6 +++
.../partition/HashBasedNumberedShardSpec.java | 6 +++
.../timeline/partition/HashBucketShardSpec.java | 8 ++-
.../druid/timeline/partition/LinearShardSpec.java | 6 +++
.../druid/timeline/partition/NoneShardSpec.java | 6 +++
.../partition/NumberedOverwriteShardSpec.java | 7 ++-
.../timeline/partition/NumberedShardSpec.java | 6 +++
.../apache/druid/timeline/partition/ShardSpec.java | 63 +++++++++++++++++-----
.../SingleDimensionRangeBucketShardSpec.java | 8 ++-
.../partition/SingleDimensionShardSpec.java | 6 +++
.../org/apache/druid/timeline/DataSegmentTest.java | 1 +
.../BuildingDimensionRangeShardSpecTest.java | 10 ++--
.../BuildingHashBasedNumberedShardSpecTest.java | 9 ++--
.../partition/BuildingNumberedShardSpecTest.java | 6 ++-
.../BuildingSingleDimensionShardSpecTest.java | 10 ++--
.../DimensionRangeBucketShardSpecTest.java | 10 ++--
.../partition/HashBasedNumberedShardSpecTest.java | 1 +
.../partition/HashBucketShardSpecTest.java | 7 ++-
.../timeline/partition/NoneShardSpecTest.java | 1 +
.../partition/NumberedOverwriteShardSpecTest.java | 6 ++-
.../timeline/partition/NumberedShardSpecTest.java | 1 +
.../SingleDimensionRangeBucketShardSpecTest.java | 7 ++-
.../partition/SingleDimensionShardSpecTest.java | 1 +
.../actions/SegmentTransactionalInsertAction.java | 4 ++
.../java/org/apache/druid/query/DruidMetrics.java | 2 +
30 files changed, 181 insertions(+), 56 deletions(-)
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java
index 166c7b8..308e364 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpec.java
@@ -39,8 +39,6 @@ import java.util.Objects;
*/
public class BuildingDimensionRangeShardSpec implements BuildingShardSpec<DimensionRangeShardSpec>
{
- public static final String TYPE = "building_range";
-
private final int bucketId;
private final List<String> dimensions;
@Nullable
@@ -125,6 +123,12 @@ public class BuildingDimensionRangeShardSpec implements BuildingShardSpec<Dimens
}
@Override
+ public String getType()
+ {
+ return Type.BUILDING_RANGE;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpec.java
index f5c43b0..98588de 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpec.java
@@ -36,8 +36,6 @@ import java.util.Objects;
*/
public class BuildingHashBasedNumberedShardSpec implements BuildingShardSpec<HashBasedNumberedShardSpec>
{
- public static final String TYPE = "building_hashed";
-
private final int partitionId;
private final int bucketId;
private final int numBuckets;
@@ -121,6 +119,12 @@ public class BuildingHashBasedNumberedShardSpec implements BuildingShardSpec<Has
}
@Override
+ public String getType()
+ {
+ return Type.BUILDING_HASHED;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpec.java
index a179d3c..52b3069 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpec.java
@@ -35,7 +35,6 @@ import java.util.Objects;
*/
public class BuildingNumberedShardSpec implements BuildingShardSpec<NumberedShardSpec>
{
- public static final String TYPE = "building_numbered";
private final int partitionId;
@@ -75,6 +74,12 @@ public class BuildingNumberedShardSpec implements BuildingShardSpec<NumberedShar
}
@Override
+ public String getType()
+ {
+ return Type.BUILDING_NUMBERED;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java
index 1b9c613..ff7d5b9 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpec.java
@@ -37,8 +37,6 @@ import java.util.Objects;
*/
public class BuildingSingleDimensionShardSpec extends BuildingDimensionRangeShardSpec
{
- public static final String TYPE = "building_single_dim";
-
private final String dimension;
@Nullable
@@ -111,6 +109,12 @@ public class BuildingSingleDimensionShardSpec extends BuildingDimensionRangeShar
}
@Override
+ public String getType()
+ {
+ return Type.BUILDING_SINGLE_DIM;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpec.java
index 17491a6..bfeb752 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpec.java
@@ -42,8 +42,6 @@ import java.util.Objects;
*/
public class DimensionRangeBucketShardSpec implements BucketNumberedShardSpec<BuildingDimensionRangeShardSpec>
{
- public static final String TYPE = "bucket_range";
-
private final int bucketId;
private final List<String> dimensions;
@Nullable
@@ -140,6 +138,12 @@ public class DimensionRangeBucketShardSpec implements BucketNumberedShardSpec<Bu
}
@Override
+ public String getType()
+ {
+ return Type.BUCKET_RANGE;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeShardSpec.java
index d4a964c..bf84508 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/DimensionRangeShardSpec.java
@@ -213,6 +213,12 @@ public class DimensionRangeShardSpec implements ShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.RANGE;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java
index 86a0554..4205a40 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpec.java
@@ -246,6 +246,12 @@ public class HashBasedNumberedShardSpec extends NumberedShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.HASHED;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/HashBucketShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/HashBucketShardSpec.java
index cef6686..21e21eb 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/HashBucketShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/HashBucketShardSpec.java
@@ -35,8 +35,6 @@ import java.util.Objects;
*/
public class HashBucketShardSpec implements BucketNumberedShardSpec<BuildingHashBasedNumberedShardSpec>
{
- public static final String TYPE = "bucket_hash";
-
private final int bucketId;
private final int numBuckets;
private final List<String> partitionDimensions;
@@ -111,6 +109,12 @@ public class HashBucketShardSpec implements BucketNumberedShardSpec<BuildingHash
}
@Override
+ public String getType()
+ {
+ return Type.BUCKET_HASH;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java
index 837c8e6..27659e5 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/LinearShardSpec.java
@@ -80,6 +80,12 @@ public final class LinearShardSpec implements ShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.LINEAR;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java
index b123c24..4661b3a 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/NoneShardSpec.java
@@ -84,6 +84,12 @@ public class NoneShardSpec implements ShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.NONE;
+ }
+
+ @Override
public boolean equals(Object obj)
{
return obj instanceof NoneShardSpec;
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java
index 808abde..563865d 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpec.java
@@ -53,7 +53,6 @@ import java.util.Objects;
*/
public class NumberedOverwriteShardSpec implements OverwriteShardSpec
{
- public static final String TYPE = "numbered_overwrite";
private final int partitionId;
private final short startRootPartitionId;
@@ -200,6 +199,12 @@ public class NumberedOverwriteShardSpec implements OverwriteShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.NUMBERED_OVERWRITE;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java
index e6ad1e4..aa3d5b3 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/NumberedShardSpec.java
@@ -107,6 +107,12 @@ public class NumberedShardSpec implements ShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.NUMBERED;
+ }
+
+ @Override
public String toString()
{
return "NumberedShardSpec{" +
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java
index 16a60c1..dda63ef 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/ShardSpec.java
@@ -33,25 +33,25 @@ import java.util.Map;
*/
@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
@JsonSubTypes({
- @JsonSubTypes.Type(name = "none", value = NoneShardSpec.class),
- @JsonSubTypes.Type(name = "single", value = SingleDimensionShardSpec.class),
- @JsonSubTypes.Type(name = "range", value = DimensionRangeShardSpec.class),
- @JsonSubTypes.Type(name = "linear", value = LinearShardSpec.class),
- @JsonSubTypes.Type(name = "numbered", value = NumberedShardSpec.class),
- @JsonSubTypes.Type(name = "hashed", value = HashBasedNumberedShardSpec.class),
- @JsonSubTypes.Type(name = NumberedOverwriteShardSpec.TYPE, value = NumberedOverwriteShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.NONE, value = NoneShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.SINGLE, value = SingleDimensionShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.RANGE, value = DimensionRangeShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.LINEAR, value = LinearShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.NUMBERED, value = NumberedShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.HASHED, value = HashBasedNumberedShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.NUMBERED_OVERWRITE, value = NumberedOverwriteShardSpec.class),
// BuildingShardSpecs are the shardSpec with missing numCorePartitions, and thus must not be published.
// See BuildingShardSpec for more details.
- @JsonSubTypes.Type(name = BuildingNumberedShardSpec.TYPE, value = BuildingNumberedShardSpec.class),
- @JsonSubTypes.Type(name = BuildingHashBasedNumberedShardSpec.TYPE, value = BuildingHashBasedNumberedShardSpec.class),
- @JsonSubTypes.Type(name = BuildingSingleDimensionShardSpec.TYPE, value = BuildingSingleDimensionShardSpec.class),
- @JsonSubTypes.Type(name = BuildingDimensionRangeShardSpec.TYPE, value = BuildingDimensionRangeShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUILDING_NUMBERED, value = BuildingNumberedShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUILDING_HASHED, value = BuildingHashBasedNumberedShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUILDING_SINGLE_DIM, value = BuildingSingleDimensionShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUILDING_RANGE, value = BuildingDimensionRangeShardSpec.class),
// BucketShardSpecs are the shardSpec with missing partitionId and numCorePartitions.
// These shardSpecs must not be used in segment push.
// See BucketShardSpec for more details.
- @JsonSubTypes.Type(name = HashBucketShardSpec.TYPE, value = HashBucketShardSpec.class),
- @JsonSubTypes.Type(name = SingleDimensionRangeBucketShardSpec.TYPE, value = SingleDimensionRangeBucketShardSpec.class),
- @JsonSubTypes.Type(name = DimensionRangeBucketShardSpec.TYPE, value = DimensionRangeBucketShardSpec.class)
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUCKET_HASH, value = HashBucketShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUCKET_SINGLE_DIM, value = SingleDimensionRangeBucketShardSpec.class),
+ @JsonSubTypes.Type(name = ShardSpec.Type.BUCKET_RANGE, value = DimensionRangeBucketShardSpec.class)
})
public interface ShardSpec
{
@@ -124,6 +124,15 @@ public interface ShardSpec
boolean possibleInDomain(Map<String, RangeSet<String>> domain);
/**
+ * Get the type name of this ShardSpec.
+ */
+ @JsonIgnore
+ default String getType()
+ {
+ return Type.UNKNOWN;
+ }
+
+ /**
* Returns true if this shardSpec and the given {@link PartialShardSpec} share the same partition space.
* All shardSpecs except {@link OverwriteShardSpec} use the root-generation partition space and thus share the same
* space.
@@ -134,4 +143,30 @@ public interface ShardSpec
{
return !partialShardSpec.useNonRootGenerationPartitionSpace();
}
+
+ /**
+ * ShardSpec type names.
+ */
+ interface Type
+ {
+ String UNKNOWN = "unknown";
+ String NONE = "none";
+
+ String SINGLE = "single";
+ String RANGE = "range";
+ String LINEAR = "linear";
+ String NUMBERED = "numbered";
+ String HASHED = "hashed";
+
+ String NUMBERED_OVERWRITE = "numbered_overwrite";
+
+ String BUILDING_NUMBERED = "building_numbered";
+ String BUILDING_HASHED = "building_hashed";
+ String BUILDING_SINGLE_DIM = "building_single_dim";
+ String BUILDING_RANGE = "building_range";
+
+ String BUCKET_HASH = "bucket_hash";
+ String BUCKET_SINGLE_DIM = "bucket_single_dim";
+ String BUCKET_RANGE = "bucket_range";
+ }
}
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpec.java
index 1ee236d..b90d8b8 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpec.java
@@ -35,8 +35,6 @@ import java.util.Objects;
*/
public class SingleDimensionRangeBucketShardSpec implements BucketNumberedShardSpec<BuildingSingleDimensionShardSpec>
{
- public static final String TYPE = "bucket_single_dim";
-
private final int bucketId;
private final String dimension;
@Nullable
@@ -110,6 +108,12 @@ public class SingleDimensionRangeBucketShardSpec implements BucketNumberedShardS
}
@Override
+ public String getType()
+ {
+ return Type.BUCKET_SINGLE_DIM;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java
index e4bff90..700af53 100644
--- a/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java
+++ b/core/src/main/java/org/apache/druid/timeline/partition/SingleDimensionShardSpec.java
@@ -204,6 +204,12 @@ public class SingleDimensionShardSpec extends DimensionRangeShardSpec
}
@Override
+ public String getType()
+ {
+ return Type.SINGLE;
+ }
+
+ @Override
public boolean equals(Object o)
{
if (this == o) {
diff --git a/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java b/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java
index 0ed35da..71c4d90 100644
--- a/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/DataSegmentTest.java
@@ -93,6 +93,7 @@ public class DataSegmentTest
{
return true;
}
+
};
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java
index 4c8045a..b8099e2 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingDimensionRangeShardSpecTest.java
@@ -89,7 +89,7 @@ public class BuildingDimensionRangeShardSpecTest
{
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
mapper.registerSubtypes(
- new NamedType(BuildingDimensionRangeShardSpec.class, BuildingDimensionRangeShardSpec.TYPE)
+ new NamedType(BuildingDimensionRangeShardSpec.class, ShardSpec.Type.BUILDING_RANGE)
);
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
final BuildingDimensionRangeShardSpec original = new BuildingDimensionRangeShardSpec(
@@ -100,10 +100,10 @@ public class BuildingDimensionRangeShardSpecTest
5
);
final String json = mapper.writeValueAsString(original);
- final BuildingDimensionRangeShardSpec fromJson = (BuildingDimensionRangeShardSpec) mapper.readValue(
- json,
- ShardSpec.class
- );
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUILDING_RANGE, shardSpec.getType());
+
+ final BuildingDimensionRangeShardSpec fromJson = (BuildingDimensionRangeShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpecTest.java
index 6262169..1722560 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingHashBasedNumberedShardSpecTest.java
@@ -68,7 +68,7 @@ public class BuildingHashBasedNumberedShardSpecTest
public void testSerde() throws JsonProcessingException
{
mapper.registerSubtypes(
- new NamedType(BuildingHashBasedNumberedShardSpec.class, BuildingHashBasedNumberedShardSpec.TYPE)
+ new NamedType(BuildingHashBasedNumberedShardSpec.class, ShardSpec.Type.BUILDING_HASHED)
);
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
final BuildingHashBasedNumberedShardSpec original = new BuildingHashBasedNumberedShardSpec(
@@ -80,10 +80,9 @@ public class BuildingHashBasedNumberedShardSpecTest
mapper
);
final String json = mapper.writeValueAsString(original);
- final BuildingHashBasedNumberedShardSpec fromJson = (BuildingHashBasedNumberedShardSpec) mapper.readValue(
- json,
- ShardSpec.class
- );
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUILDING_HASHED, shardSpec.getType());
+ final BuildingHashBasedNumberedShardSpec fromJson = (BuildingHashBasedNumberedShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpecTest.java
index b608d4c..21ff72c 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingNumberedShardSpecTest.java
@@ -49,10 +49,12 @@ public class BuildingNumberedShardSpecTest
{
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
- mapper.registerSubtypes(new NamedType(BuildingNumberedShardSpec.class, BuildingNumberedShardSpec.TYPE));
+ mapper.registerSubtypes(new NamedType(BuildingNumberedShardSpec.class, ShardSpec.Type.BUILDING_NUMBERED));
final BuildingNumberedShardSpec original = new BuildingNumberedShardSpec(5);
final String json = mapper.writeValueAsString(original);
- final BuildingNumberedShardSpec fromJson = (BuildingNumberedShardSpec) mapper.readValue(json, ShardSpec.class);
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUILDING_NUMBERED, shardSpec.getType());
+ final BuildingNumberedShardSpec fromJson = (BuildingNumberedShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java
index 84ff681..bf78ceb 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/BuildingSingleDimensionShardSpecTest.java
@@ -56,8 +56,10 @@ public class BuildingSingleDimensionShardSpecTest
final BuildingSingleDimensionShardSpec original =
new BuildingSingleDimensionShardSpec(1, "dim", "start", "end", 5);
final String json = serialize(original);
- final BuildingSingleDimensionShardSpec fromJson =
- (BuildingSingleDimensionShardSpec) deserialize(json, ShardSpec.class);
+ ShardSpec shardSpec = deserialize(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUILDING_SINGLE_DIM, shardSpec.getType());
+
+ final BuildingSingleDimensionShardSpec fromJson = (BuildingSingleDimensionShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
@@ -80,7 +82,7 @@ public class BuildingSingleDimensionShardSpecTest
@Test
public void testDeserializeFromMap()
{
- final String json = "{\"type\": \"" + BuildingSingleDimensionShardSpec.TYPE + "\","
+ final String json = "{\"type\": \"" + ShardSpec.Type.BUILDING_SINGLE_DIM + "\","
+ " \"bucketId\":1,"
+ " \"dimension\": \"dim\","
+ " \"start\": \"abc\","
@@ -108,7 +110,7 @@ public class BuildingSingleDimensionShardSpecTest
{
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
mapper.registerSubtypes(
- new NamedType(BuildingSingleDimensionShardSpec.class, BuildingSingleDimensionShardSpec.TYPE)
+ new NamedType(BuildingSingleDimensionShardSpec.class, ShardSpec.Type.BUILDING_SINGLE_DIM)
);
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java
index 16d34dd..f4a2d88 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/DimensionRangeBucketShardSpecTest.java
@@ -148,7 +148,7 @@ public class DimensionRangeBucketShardSpecTest
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
mapper.registerSubtypes(new NamedType(
DimensionRangeBucketShardSpec.class,
- DimensionRangeBucketShardSpec.TYPE
+ ShardSpec.Type.BUCKET_RANGE
));
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
final DimensionRangeBucketShardSpec original = new DimensionRangeBucketShardSpec(
@@ -158,11 +158,9 @@ public class DimensionRangeBucketShardSpecTest
StringTuple.create("end1", "end2")
);
final String json = mapper.writeValueAsString(original);
- final DimensionRangeBucketShardSpec fromJson = (DimensionRangeBucketShardSpec) mapper.readValue(
- json,
- ShardSpec.class
- );
- Assert.assertEquals(original, fromJson);
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUCKET_RANGE, shardSpec.getType());
+ Assert.assertEquals(original, shardSpec);
}
@Test
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java
index 74a2bb1..c3cceae 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBasedNumberedShardSpecTest.java
@@ -99,6 +99,7 @@ public class HashBasedNumberedShardSpecTest
"{\"type\": \"hashed\", \"partitions\": 2, \"partitionNum\": 1, \"partitionDimensions\":[\"visitor_id\"]}",
ShardSpec.class
);
+ Assert.assertEquals(ShardSpec.Type.HASHED, specWithPartitionDimensions.getType());
Assert.assertEquals(1, specWithPartitionDimensions.getPartitionNum());
Assert.assertEquals(2, specWithPartitionDimensions.getNumCorePartitions());
Assert.assertEquals(2, ((HashBasedNumberedShardSpec) specWithPartitionDimensions).getNumBuckets());
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/HashBucketShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/HashBucketShardSpecTest.java
index 146157b..2f1b2ef 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/HashBucketShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/HashBucketShardSpecTest.java
@@ -113,7 +113,7 @@ public class HashBucketShardSpecTest
@Test
public void testSerde() throws JsonProcessingException
{
- mapper.registerSubtypes(new NamedType(HashBucketShardSpec.class, HashBucketShardSpec.TYPE));
+ mapper.registerSubtypes(new NamedType(HashBucketShardSpec.class, ShardSpec.Type.BUCKET_HASH));
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
final HashBucketShardSpec original = new HashBucketShardSpec(
@@ -124,7 +124,10 @@ public class HashBucketShardSpecTest
mapper
);
final String json = mapper.writeValueAsString(original);
- final HashBucketShardSpec fromJson = (HashBucketShardSpec) mapper.readValue(json, ShardSpec.class);
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUCKET_HASH, shardSpec.getType());
+
+ final HashBucketShardSpec fromJson = (HashBucketShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/NoneShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/NoneShardSpecTest.java
index a277773..65f17b7 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/NoneShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/NoneShardSpecTest.java
@@ -48,6 +48,7 @@ public class NoneShardSpecTest
// Serde should return same object instead of creating new one every time.
Assert.assertTrue(serde1 == serde2);
Assert.assertTrue(one == serde1);
+ Assert.assertEquals(ShardSpec.Type.NONE, serde1.getType());
}
@Test
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpecTest.java
index 4079143..d35364c 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/NumberedOverwriteShardSpecTest.java
@@ -38,7 +38,7 @@ public class NumberedOverwriteShardSpecTest
public void testSerde() throws JsonProcessingException
{
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
- mapper.registerSubtypes(new NamedType(NumberedOverwriteShardSpec.class, NumberedOverwriteShardSpec.TYPE));
+ mapper.registerSubtypes(new NamedType(NumberedOverwriteShardSpec.class, ShardSpec.Type.NUMBERED_OVERWRITE));
final NumberedOverwriteShardSpec original = new NumberedOverwriteShardSpec(
PartitionIds.NON_ROOT_GEN_START_PARTITION_ID + 2,
0,
@@ -47,7 +47,9 @@ public class NumberedOverwriteShardSpecTest
(short) 3
);
final String json = mapper.writeValueAsString(original);
- final NumberedOverwriteShardSpec fromJson = (NumberedOverwriteShardSpec) mapper.readValue(json, ShardSpec.class);
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.NUMBERED_OVERWRITE, shardSpec.getType());
+ final NumberedOverwriteShardSpec fromJson = (NumberedOverwriteShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java
index 1c77a7f..4a3ba4c 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/NumberedShardSpecTest.java
@@ -58,6 +58,7 @@ public class NumberedShardSpecTest
);
Assert.assertEquals(1, spec.getPartitionNum());
Assert.assertEquals(2, spec.getNumCorePartitions());
+ Assert.assertEquals(ShardSpec.Type.NUMBERED, spec.getType());
}
@Test
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpecTest.java
index fb8d5f4..acecc64 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionRangeBucketShardSpecTest.java
@@ -99,11 +99,14 @@ public class SingleDimensionRangeBucketShardSpecTest
public void testSerde() throws JsonProcessingException
{
final ObjectMapper mapper = ShardSpecTestUtils.initObjectMapper();
- mapper.registerSubtypes(new NamedType(SingleDimensionRangeBucketShardSpec.class, SingleDimensionRangeBucketShardSpec.TYPE));
+ mapper.registerSubtypes(new NamedType(SingleDimensionRangeBucketShardSpec.class, ShardSpec.Type.BUCKET_SINGLE_DIM));
mapper.setInjectableValues(new Std().addValue(ObjectMapper.class, mapper));
final SingleDimensionRangeBucketShardSpec original = new SingleDimensionRangeBucketShardSpec(1, "dim", "start", "end");
final String json = mapper.writeValueAsString(original);
- final SingleDimensionRangeBucketShardSpec fromJson = (SingleDimensionRangeBucketShardSpec) mapper.readValue(json, ShardSpec.class);
+ ShardSpec shardSpec = mapper.readValue(json, ShardSpec.class);
+ Assert.assertEquals(ShardSpec.Type.BUCKET_SINGLE_DIM, shardSpec.getType());
+
+ final SingleDimensionRangeBucketShardSpec fromJson = (SingleDimensionRangeBucketShardSpec) shardSpec;
Assert.assertEquals(original, fromJson);
}
diff --git a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java
index 219c0ea..9b14450 100644
--- a/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java
+++ b/core/src/test/java/org/apache/druid/timeline/partition/SingleDimensionShardSpecTest.java
@@ -175,6 +175,7 @@ public class SingleDimensionShardSpecTest
+ "\"numCorePartitions\": 10}";
ShardSpec shardSpec = OBJECT_MAPPER.readValue(json, ShardSpec.class);
Assert.assertTrue(shardSpec instanceof SingleDimensionShardSpec);
+ Assert.assertEquals(ShardSpec.Type.SINGLE, shardSpec.getType());
SingleDimensionShardSpec singleDimShardSpec = (SingleDimensionShardSpec) shardSpec;
Assert.assertEquals(
diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
index 1e0df0e..233739e 100644
--- a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
+++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
@@ -252,6 +252,10 @@ public class SegmentTransactionalInsertAction implements TaskAction<SegmentPubli
// getSegments() should return an empty set if announceHistoricalSegments() failed
for (DataSegment segment : retVal.getSegments()) {
metricBuilder.setDimension(DruidMetrics.INTERVAL, segment.getInterval().toString());
+ metricBuilder.setDimension(
+ DruidMetrics.PARTITIONING_TYPE,
+ segment.getShardSpec() == null ? null : segment.getShardSpec().getType()
+ );
toolbox.getEmitter().emit(metricBuilder.build("segment/added/bytes", segment.getSize()));
}
diff --git a/processing/src/main/java/org/apache/druid/query/DruidMetrics.java b/processing/src/main/java/org/apache/druid/query/DruidMetrics.java
index 9d6aa58..44338af 100644
--- a/processing/src/main/java/org/apache/druid/query/DruidMetrics.java
+++ b/processing/src/main/java/org/apache/druid/query/DruidMetrics.java
@@ -35,6 +35,8 @@ public class DruidMetrics
public static final String TASK_ID = "taskId";
public static final String STATUS = "status";
+ public static final String PARTITIONING_TYPE = "partitioningType";
+
// task metrics
public static final String TASK_TYPE = "taskType";
public static final String TASK_STATUS = "taskStatus";
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org