You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by xv...@apache.org on 2020/03/20 07:22:20 UTC
[druid] 01/01: support schemaless ingestion for transformed
dimensions
This is an automated email from the ASF dual-hosted git repository.
xvrl pushed a commit to branch transformspec-add-dims
in repository https://gitbox.apache.org/repos/asf/druid.git
commit b154180065ea76e7b19e84d90d94ebf2a8cd5375
Author: Xavier Léauté <xv...@apache.org>
AuthorDate: Fri Mar 20 00:21:38 2020 -0700
support schemaless ingestion for transformed dimensions
fixes #7952
---
.../firehose/IngestSegmentFirehoseFactoryTest.java | 2 +-
.../apache/druid/segment/transform/Transformer.java | 19 ++++++++++++++++++-
.../druid/segment/indexing/TransformSpecTest.java | 2 +-
3 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java
index 1a786ec..8a7824d 100644
--- a/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java
+++ b/indexing-service/src/test/java/org/apache/druid/indexing/firehose/IngestSegmentFirehoseFactoryTest.java
@@ -509,7 +509,7 @@ public class IngestSegmentFirehoseFactoryTest
skipped++;
continue;
}
- Assert.assertArrayEquals(new String[]{DIM_NAME}, row.getDimensions().toArray());
+ Assert.assertArrayEquals(new String[]{DIM_NAME, METRIC_FLOAT_NAME}, row.getDimensions().toArray());
Assert.assertArrayEquals(new String[]{DIM_VALUE}, row.getDimension(DIM_NAME).toArray());
Assert.assertEquals(METRIC_LONG_VALUE.longValue(), row.getMetric(METRIC_LONG_NAME).longValue());
Assert.assertEquals(
diff --git a/processing/src/main/java/org/apache/druid/segment/transform/Transformer.java b/processing/src/main/java/org/apache/druid/segment/transform/Transformer.java
index 48b0421..9d350cf 100644
--- a/processing/src/main/java/org/apache/druid/segment/transform/Transformer.java
+++ b/processing/src/main/java/org/apache/druid/segment/transform/Transformer.java
@@ -19,6 +19,7 @@
package org.apache.druid.segment.transform;
+import com.google.common.collect.Sets;
import org.apache.druid.data.input.InputRow;
import org.apache.druid.data.input.InputRowListPlusRawValues;
import org.apache.druid.data.input.Row;
@@ -37,6 +38,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
/**
*
@@ -138,17 +140,32 @@ public class Transformer
{
private final InputRow row;
private final Map<String, RowFunction> transforms;
+ private final List<String> dimensions;
public TransformedInputRow(final InputRow row, final Map<String, RowFunction> transforms)
{
this.row = row;
this.transforms = transforms;
+
+ Set<String> transformedDims = Sets.newHashSet(transforms.keySet());
+
+ for (String dim : row.getDimensions()) {
+ transformedDims.remove(dim);
+ }
+
+ if (transformedDims.isEmpty()) {
+ this.dimensions = row.getDimensions();
+ } else {
+ this.dimensions = new ArrayList<>(row.getDimensions().size() + transforms.size());
+ dimensions.addAll(row.getDimensions());
+ dimensions.addAll(transformedDims);
+ }
}
@Override
public List<String> getDimensions()
{
- return row.getDimensions();
+ return dimensions;
}
@Override
diff --git a/server/src/test/java/org/apache/druid/segment/indexing/TransformSpecTest.java b/server/src/test/java/org/apache/druid/segment/indexing/TransformSpecTest.java
index 8102a71..006a2f3 100644
--- a/server/src/test/java/org/apache/druid/segment/indexing/TransformSpecTest.java
+++ b/server/src/test/java/org/apache/druid/segment/indexing/TransformSpecTest.java
@@ -85,7 +85,7 @@ public class TransformSpecTest
Assert.assertNotNull(row);
Assert.assertEquals(DateTimes.of("2000-01-01").getMillis(), row.getTimestampFromEpoch());
Assert.assertEquals(DateTimes.of("2000-01-01"), row.getTimestamp());
- Assert.assertEquals(ImmutableList.of("f", "x", "y"), row.getDimensions());
+ Assert.assertEquals(ImmutableList.of("f", "x", "y", "g", "h"), row.getDimensions());
Assert.assertEquals(ImmutableList.of("foo"), row.getDimension("x"));
Assert.assertEquals(3.0, row.getMetric("b").doubleValue(), 0);
Assert.assertEquals("foobar", row.getRaw("f"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org