You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2023/03/28 21:44:51 UTC
[druid] branch master updated: add backwards compat mode for frontCoded stringEncodingStrategy (#13988)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 2219e68fa3 add backwards compat mode for frontCoded stringEncodingStrategy (#13988)
2219e68fa3 is described below
commit 2219e68fa3c602eaad8f5ba84acc36f8e2cabebc
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Tue Mar 28 14:44:44 2023 -0700
add backwards compat mode for frontCoded stringEncodingStrategy (#13988)
---
.../benchmark/FrontCodedIndexedBenchmark.java | 4 +-
docs/ingestion/ingestion-spec.md | 4 +-
.../segment/column/StringEncodingStrategies.java | 2 +-
.../segment/column/StringEncodingStrategy.java | 29 ++++++--
.../druid/segment/data/FrontCodedIndexed.java | 20 +++++-
.../segment/data/FrontCodedIndexedWriter.java | 33 ++++-----
.../segment/column/StringEncodingStrategyTest.java | 80 ++++++++++++++++++++++
.../druid/segment/data/FrontCodedIndexedTest.java | 49 +++++++------
8 files changed, 168 insertions(+), 53 deletions(-)
diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
index 00be920253..fe0e717ab1 100644
--- a/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
+++ b/benchmarks/src/test/java/org/apache/druid/benchmark/FrontCodedIndexedBenchmark.java
@@ -139,7 +139,7 @@ public class FrontCodedIndexedBenchmark
new OnHeapMemorySegmentWriteOutMedium(),
ByteOrder.nativeOrder(),
"front-coded-4".equals(indexType) ? 4 : 16,
- false
+ FrontCodedIndexed.V0
);
frontCodedIndexedWriter.open();
@@ -147,7 +147,7 @@ public class FrontCodedIndexedBenchmark
new OnHeapMemorySegmentWriteOutMedium(),
ByteOrder.nativeOrder(),
"front-coded-incremental-buckets-4".equals(indexType) ? 4 : 16,
- true
+ FrontCodedIndexed.V1
);
frontCodedIndexedWriterIncrementalBuckets.open();
diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index d0ae34eec3..22e4d167bb 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -477,7 +477,7 @@ The `indexSpec` object can include the following properties:
|-----|-----------|-------|
|bitmap|Compression format for bitmap indexes. Should be a JSON object with `type` set to `roaring` or `concise`.|`{"type": "roaring"}`|
|dimensionCompression|Compression format for dimension columns. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
-|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /><br />See [Front coding](#front-coding) for more information.|`{"type":"utf8"}`|
+|stringDictionaryEncoding|Encoding format for STRING value dictionaries used by STRING and COMPLEX<json> columns. <br /><br />Example to enable front coding: `{"type":"frontCoded", "bucketSize": 4}`<br />`bucketSize` is the number of values to place in a bucket to perform delta encoding. Must be a power of 2, maximum is 128. Defaults to 4.<br /> `formatVersion` can specify older versions for backwards compatibility during rolling upgrades, valid options are `0` and `1`. Defaults to [...]
|metricCompression|Compression format for primitive type metric columns. Options are `lz4`, `lzf`, `zstd`, `uncompressed`, or `none` (which is more efficient than `uncompressed`, but not supported by older versions of Druid).|`lz4`|
|longEncoding|Encoding format for long-typed columns. Applies regardless of whether they are dimensions or metrics. Options are `auto` or `longs`. `auto` encodes the values using offset or lookup table depending on column cardinality, and store them with variable size. `longs` stores the value as-is with 8 bytes each.|`longs`|
|jsonCompression|Compression format to use for nested column raw data. Options are `lz4`, `lzf`, `zstd`, or `uncompressed`.|`lz4`|
@@ -488,7 +488,7 @@ Front coding is an experimental feature starting in version 25.0. Front coding i
You can enable front coding with all types of ingestion. For information on defining an `indexSpec` in a query context, see [SQL-based ingestion reference](../multi-stage-query/reference.md#context-parameters).
-> Front coding is new to Druid 25.0 so the current recommendation is to enable it in a staging environment and fully test your use case before using in production. Segments created with front coding enabled are not compatible with Druid versions older than 25.0.
+> Front coding was originally introduced in Druid 25.0, and an improved 'version 1' was introduced in Druid 26.0, with typically faster read speed and smaller storage size. The current recommendation is to enable it in a staging environment and fully test your use case before using in production. By default, segments created with front coding enabled in Druid 26.0 are not backwards compatible with Druid 25.0, and those created with Druid 25.0 are not compatible with Druid versions older [...]
Beyond these properties, each ingestion method has its own specific tuning properties. See the documentation for each
[ingestion method](./index.md#ingestion-methods) for details.
diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
index 0c04f0288c..c4a9b0661a 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategies.java
@@ -58,7 +58,7 @@ public class StringEncodingStrategies
writeoutMedium,
IndexIO.BYTE_ORDER,
strategy.getBucketSize(),
- true
+ strategy.getFormatVersion()
);
} else {
throw new ISE("Unknown encoding strategy: %s", encodingStrategy.getType());
diff --git a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
index e850b0c362..a8246c9d75 100644
--- a/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
+++ b/processing/src/main/java/org/apache/druid/segment/column/StringEncodingStrategy.java
@@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonSubTypes;
import com.fasterxml.jackson.annotation.JsonTypeInfo;
import org.apache.druid.java.util.common.ISE;
+import org.apache.druid.segment.data.FrontCodedIndexed;
import javax.annotation.Nullable;
import java.util.Objects;
@@ -84,20 +85,31 @@ public interface StringEncodingStrategy
class FrontCoded implements StringEncodingStrategy
{
- public static final int DEFAULT_BUCKET_SIZE = 4;
@JsonProperty
private final int bucketSize;
+ @JsonProperty
+ private final byte formatVersion;
+
@JsonCreator
public FrontCoded(
- @JsonProperty("bucketSize") @Nullable Integer bucketSize
+ @JsonProperty("bucketSize") @Nullable Integer bucketSize,
+ @JsonProperty("formatVersion") @Nullable Byte version
)
{
- this.bucketSize = bucketSize == null ? DEFAULT_BUCKET_SIZE : bucketSize;
+ this.bucketSize = bucketSize == null ? FrontCodedIndexed.DEFAULT_BUCKET_SIZE : bucketSize;
if (Integer.bitCount(this.bucketSize) != 1) {
throw new ISE("bucketSize must be a power of two but was[%,d]", bucketSize);
}
+ this.formatVersion = version == null
+ ? FrontCodedIndexed.DEFAULT_VERSION
+ : FrontCodedIndexed.validateVersion(version);
+ }
+
+ public FrontCoded(@Nullable Integer bucketSize)
+ {
+ this(bucketSize, null);
}
@JsonProperty
@@ -106,6 +118,12 @@ public interface StringEncodingStrategy
return bucketSize;
}
+ @JsonProperty
+ public byte getFormatVersion()
+ {
+ return formatVersion;
+ }
+
@Override
public String getType()
{
@@ -128,13 +146,13 @@ public interface StringEncodingStrategy
return false;
}
FrontCoded that = (FrontCoded) o;
- return bucketSize == that.bucketSize;
+ return bucketSize == that.bucketSize && formatVersion == that.formatVersion;
}
@Override
public int hashCode()
{
- return Objects.hash(bucketSize);
+ return Objects.hash(bucketSize, formatVersion);
}
@Override
@@ -142,6 +160,7 @@ public interface StringEncodingStrategy
{
return "FrontCoded{" +
"bucketSize=" + bucketSize +
+ ", formatVersion=" + formatVersion +
'}';
}
}
diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
index 27a84b5df3..bb135b0b9a 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexed.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import org.apache.druid.annotations.SuppressFBWarnings;
import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.java.util.common.IAE;
import org.apache.druid.java.util.common.ISE;
import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector;
@@ -76,11 +77,28 @@ import java.util.NoSuchElementException;
*/
public final class FrontCodedIndexed implements Indexed<ByteBuffer>
{
+ public static final byte V0 = 0;
+ public static final byte V1 = 1;
+ public static final byte DEFAULT_VERSION = V1;
+ public static final int DEFAULT_BUCKET_SIZE = 4;
+
+ public static byte validateVersion(byte version)
+ {
+ if (version != FrontCodedIndexed.V0 && version != FrontCodedIndexed.V1) {
+ throw new IAE(
+ "Unknown format version for FrontCodedIndexed [%s], must be [%s] or [%s]",
+ version,
+ FrontCodedIndexed.V0,
+ FrontCodedIndexed.V1
+ );
+ }
+ return version;
+ }
public static Supplier<FrontCodedIndexed> read(ByteBuffer buffer, ByteOrder ordering)
{
final ByteBuffer orderedBuffer = buffer.asReadOnlyBuffer().order(ordering);
final byte version = orderedBuffer.get();
- Preconditions.checkArgument(version == 0 || version == 1, "only V0 and V1 exist, encountered " + version);
+ Preconditions.checkArgument(version == V0 || version == V1, "only V0 and V1 exist, encountered " + version);
final int bucketSize = Byte.toUnsignedInt(orderedBuffer.get());
final boolean hasNull = NullHandling.IS_NULL_BYTE == orderedBuffer.get();
final int numValues = VByte.readInt(orderedBuffer);
diff --git a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
index c5a26f3a59..8f711df610 100644
--- a/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
+++ b/processing/src/main/java/org/apache/druid/segment/data/FrontCodedIndexedWriter.java
@@ -59,7 +59,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
private final byte[][] bucketBuffer;
private final ByteBuffer getOffsetBuffer;
private final int div;
- private final boolean useIncrementalBuckets;
+ private final byte version;
@Nullable
private byte[] prevObject = null;
@@ -78,7 +78,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
SegmentWriteOutMedium segmentWriteOutMedium,
ByteOrder byteOrder,
int bucketSize,
- boolean useIncrementalBuckets
+ byte version
)
{
if (Integer.bitCount(bucketSize) != 1 || bucketSize < 1 || bucketSize > 128) {
@@ -91,7 +91,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
this.bucketBuffer = new byte[bucketSize][];
this.getOffsetBuffer = ByteBuffer.allocate(Integer.BYTES).order(byteOrder);
this.div = Integer.numberOfTrailingZeros(bucketSize);
- this.useIncrementalBuckets = useIncrementalBuckets;
+ this.version = FrontCodedIndexed.validateVersion(version);
}
@Override
@@ -124,9 +124,9 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
int written;
// write the bucket, growing scratch buffer as necessary
do {
- written = useIncrementalBuckets
- ? writeIncrementalBucket(scratch, bucketBuffer, bucketSize)
- : writeBucket(scratch, bucketBuffer, bucketSize);
+ written = version == FrontCodedIndexed.V1
+ ? writeBucketV1(scratch, bucketBuffer, bucketSize)
+ : writeBucketV0(scratch, bucketBuffer, bucketSize);
if (written < 0) {
growScratch();
}
@@ -170,14 +170,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
flush();
}
resetScratch();
-
- if (useIncrementalBuckets) {
- // version 1 is incremental buckets
- scratch.put((byte) 1);
- } else {
- // version 0 all values are prefixed on first bucket value
- scratch.put((byte) 0);
- }
+ scratch.put(version);
scratch.put((byte) bucketSize);
scratch.put(hasNulls ? NullHandling.IS_NULL_BYTE : NullHandling.IS_NOT_NULL_BYTE);
VByte.writeInt(scratch, numWritten);
@@ -222,7 +215,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
final ByteBuffer bucketBuffer = ByteBuffer.allocate(bucketBytesSize).order(byteOrder);
valuesOut.readFully(startOffset, bucketBuffer);
bucketBuffer.clear();
- final ByteBuffer valueBuffer = useIncrementalBuckets
+ final ByteBuffer valueBuffer = version == FrontCodedIndexed.V1
? getFromBucketV1(bucketBuffer, relativeIndex, bucketSize)
: FrontCodedIndexed.getFromBucketV0(bucketBuffer, relativeIndex);
final byte[] valueBytes = new byte[valueBuffer.limit() - valueBuffer.position()];
@@ -248,9 +241,9 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
int written;
do {
int flushSize = remainder == 0 ? bucketSize : remainder;
- written = useIncrementalBuckets
- ? writeIncrementalBucket(scratch, bucketBuffer, flushSize)
- : writeBucket(scratch, bucketBuffer, flushSize);
+ written = version == FrontCodedIndexed.V1
+ ? writeBucketV1(scratch, bucketBuffer, flushSize)
+ : writeBucketV0(scratch, bucketBuffer, flushSize);
if (written < 0) {
growScratch();
}
@@ -283,7 +276,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
*
* Uses {@link VByte} encoded integers to indicate prefix length and value length.
*/
- public static int writeBucket(ByteBuffer buffer, byte[][] values, int numValues)
+ public static int writeBucketV0(ByteBuffer buffer, byte[][] values, int numValues)
{
int written = 0;
byte[] first = null;
@@ -333,7 +326,7 @@ public class FrontCodedIndexedWriter implements DictionaryWriter<byte[]>
*
* Uses {@link VByte} encoded integers to indicate prefix length and value length.
*/
- public static int writeIncrementalBucket(ByteBuffer buffer, byte[][] values, int numValues)
+ public static int writeBucketV1(ByteBuffer buffer, byte[][] values, int numValues)
{
int written = 0;
byte[] prev = null;
diff --git a/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java
new file mode 100644
index 0000000000..e2b219c052
--- /dev/null
+++ b/processing/src/test/java/org/apache/druid/segment/column/StringEncodingStrategyTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.segment.column;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import nl.jqno.equalsverifier.EqualsVerifier;
+import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.data.FrontCodedIndexed;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class StringEncodingStrategyTest
+{
+ private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
+
+
+ @Test
+ public void testUtf8Serde() throws JsonProcessingException
+ {
+ StringEncodingStrategy utf8 = new StringEncodingStrategy.Utf8();
+ String there = JSON_MAPPER.writeValueAsString(utf8);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+ Assert.assertEquals(utf8, andBackAgain);
+ }
+
+ @Test
+ public void testFrontCodedDefaultSerde() throws JsonProcessingException
+ {
+ StringEncodingStrategy frontCoded = new StringEncodingStrategy.FrontCoded(null, null);
+ String there = JSON_MAPPER.writeValueAsString(frontCoded);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+ Assert.assertEquals(frontCoded, andBackAgain);
+ Assert.assertEquals(FrontCodedIndexed.DEFAULT_BUCKET_SIZE, ((StringEncodingStrategy.FrontCoded) andBackAgain).getBucketSize());
+ Assert.assertEquals(FrontCodedIndexed.DEFAULT_VERSION, ((StringEncodingStrategy.FrontCoded) andBackAgain).getFormatVersion());
+
+ // this next assert seems silly, but its a sanity check to make us think hard before changing the default version,
+ // to make us think of the backwards compatibility implications, as new versions of segment format stuff cannot be
+ // downgraded to older versions of Druid and still read
+ Assert.assertEquals(FrontCodedIndexed.V1, FrontCodedIndexed.DEFAULT_VERSION);
+ }
+
+ @Test
+ public void testFrontCodedFormatSerde() throws JsonProcessingException
+ {
+ StringEncodingStrategy frontCodedV0 = new StringEncodingStrategy.FrontCoded(16, FrontCodedIndexed.V0);
+ String there = JSON_MAPPER.writeValueAsString(frontCodedV0);
+ StringEncodingStrategy andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+ Assert.assertEquals(frontCodedV0, andBackAgain);
+
+ StringEncodingStrategy frontCodedV1 = new StringEncodingStrategy.FrontCoded(8, FrontCodedIndexed.V1);
+ there = JSON_MAPPER.writeValueAsString(frontCodedV1);
+ andBackAgain = JSON_MAPPER.readValue(there, StringEncodingStrategy.class);
+ Assert.assertEquals(frontCodedV1, andBackAgain);
+ }
+
+ @Test
+ public void testEqualsAndHashcode()
+ {
+ EqualsVerifier.forClass(StringEncodingStrategy.Utf8.class).usingGetClass().verify();
+ EqualsVerifier.forClass(StringEncodingStrategy.FrontCoded.class).usingGetClass().verify();
+ }
+}
diff --git a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
index 4c1c51231a..98e6b4bbbf 100644
--- a/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
+++ b/processing/src/test/java/org/apache/druid/segment/data/FrontCodedIndexedTest.java
@@ -47,20 +47,20 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
public static Collection<Object[]> constructorFeeder()
{
return ImmutableList.of(
- new Object[]{ByteOrder.LITTLE_ENDIAN, true},
- new Object[]{ByteOrder.LITTLE_ENDIAN, false},
- new Object[]{ByteOrder.BIG_ENDIAN, true},
- new Object[]{ByteOrder.BIG_ENDIAN, false}
+ new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V1},
+ new Object[]{ByteOrder.LITTLE_ENDIAN, FrontCodedIndexed.V0},
+ new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V1},
+ new Object[]{ByteOrder.BIG_ENDIAN, FrontCodedIndexed.V0}
);
}
private final ByteOrder order;
- private final boolean useIncrementalBuckets;
+ private final byte version;
- public FrontCodedIndexedTest(ByteOrder byteOrder, boolean useIncrementalBuckets)
+ public FrontCodedIndexedTest(ByteOrder byteOrder, byte version)
{
this.order = byteOrder;
- this.useIncrementalBuckets = useIncrementalBuckets;
+ this.version = version;
}
@Test
@@ -68,7 +68,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -99,7 +99,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
- persistToBuffer(buffer, theList, 16, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 16, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -137,7 +137,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
}
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -173,7 +173,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
for (int i = 0; i < sizeBase + sizeAdjust; i++) {
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
}
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -207,7 +207,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = ImmutableList.of("hello", "helloo", "hellooo", "hellooz", "helloozy");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -231,7 +231,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
TreeSet<String> values = new TreeSet<>(GenericIndexed.STRING_STRATEGY);
values.add(null);
values.addAll(theList);
- persistToBuffer(buffer, values, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, values, 4, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
@@ -254,7 +254,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
// "\uD83D\uDCA9" and "（請參見已被刪除版本）" are a regression test for https://github.com/apache/druid/pull/13364
List<String> theList = ImmutableList.of("Győ-Moson-Sopron", "Győr", "\uD83D\uDCA9", "（請參見已被刪除版本）");
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -282,7 +282,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 12).order(order);
List<String> theList = Collections.singletonList(null);
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -308,7 +308,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
{
ByteBuffer buffer = ByteBuffer.allocate(1 << 6).order(order);
List<String> theList = Collections.emptyList();
- persistToBuffer(buffer, theList, 4, useIncrementalBuckets);
+ persistToBuffer(buffer, theList, 4, version);
buffer.position(0);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
@@ -351,7 +351,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
values.add(IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId() + IdUtils.getRandomId());
}
for (int bucketSize : bucketSizes) {
- persistToBuffer(buffer, values, bucketSize, useIncrementalBuckets);
+ persistToBuffer(buffer, values, bucketSize, version);
FrontCodedIndexed codedUtf8Indexed = FrontCodedIndexed.read(
buffer,
buffer.order()
@@ -389,7 +389,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
0,
- useIncrementalBuckets
+ version
)
);
@@ -399,7 +399,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
15,
- useIncrementalBuckets
+ version
)
);
@@ -409,12 +409,17 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
medium,
ByteOrder.nativeOrder(),
256,
- useIncrementalBuckets
+ version
)
);
}
- private static long persistToBuffer(ByteBuffer buffer, Iterable<String> sortedIterable, int bucketSize, boolean useIncrementalBuckets) throws IOException
+ private static long persistToBuffer(
+ ByteBuffer buffer,
+ Iterable<String> sortedIterable,
+ int bucketSize,
+ byte version
+ ) throws IOException
{
Iterator<String> sortedStrings = sortedIterable.iterator();
buffer.position(0);
@@ -424,7 +429,7 @@ public class FrontCodedIndexedTest extends InitializedNullHandlingTest
medium,
buffer.order(),
bucketSize,
- useIncrementalBuckets
+ version
);
writer.open();
int index = 0;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org