You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by al...@apache.org on 2022/02/02 00:55:41 UTC
[druid] branch datasketches_3_1_0 created (now 960132e)
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a change to branch datasketches_3_1_0
in repository https://gitbox.apache.org/repos/asf/druid.git.
at 960132e latest datasketches-java-3.1.0
This branch includes the following new commits:
new 960132e latest datasketches-java-3.1.0
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
[druid] 01/01: latest datasketches-java-3.1.0
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
alsay pushed a commit to branch datasketches_3_1_0
in repository https://gitbox.apache.org/repos/asf/druid.git
commit 960132e2f0f365c02ed7958ed5da0f6adb0dbcd5
Author: AlexanderSaydakov <Al...@users.noreply.github.com>
AuthorDate: Tue Feb 1 16:55:09 2022 -0800
latest datasketches-java-3.1.0
---
.../hll/HllSketchBuildBufferAggregatorHelper.java | 6 +++-
.../DoublesSketchBuildBufferAggregator.java | 2 +-
.../DoublesSketchBuildBufferAggregatorHelper.java | 6 +++-
.../DoublesSketchBuildVectorAggregator.java | 24 ++++++-------
.../quantiles/DoublesSketchMergeAggregator.java | 12 +++----
.../DoublesSketchMergeBufferAggregatorHelper.java | 6 +++-
.../DoublesSketchMergeVectorAggregator.java | 28 +++++++--------
.../quantiles/DoublesSketchAggregatorTest.java | 42 +---------------------
.../sql/DoublesSketchSqlAggregatorTest.java | 16 +++++----
.../tuple/ArrayOfDoublesSketchAggregationTest.java | 2 ++
licenses.yaml | 2 +-
pom.xml | 2 +-
12 files changed, 57 insertions(+), 91 deletions(-)
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildBufferAggregatorHelper.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildBufferAggregatorHelper.java
index 8ad8dfe..f7e386a 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildBufferAggregatorHelper.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/hll/HllSketchBuildBufferAggregatorHelper.java
@@ -23,6 +23,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
import org.apache.datasketches.hll.HllSketch;
import org.apache.datasketches.hll.TgtHllType;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import java.nio.ByteBuffer;
@@ -31,6 +33,7 @@ import java.util.IdentityHashMap;
public class HllSketchBuildBufferAggregatorHelper
{
+ private static final MemoryRequestServer MEM_REQ_SERVER = new DefaultMemoryRequestServer();
private final int lgK;
private final int size;
private final IdentityHashMap<ByteBuffer, WritableMemory> memCache = new IdentityHashMap<>();
@@ -123,7 +126,8 @@ public class HllSketchBuildBufferAggregatorHelper
private WritableMemory getMemory(final ByteBuffer buf)
{
- return memCache.computeIfAbsent(buf, b -> WritableMemory.writableWrap(b, ByteOrder.LITTLE_ENDIAN));
+ return memCache.computeIfAbsent(buf,
+ b -> WritableMemory.writableWrap(b, ByteOrder.LITTLE_ENDIAN, MEM_REQ_SERVER));
}
private void putSketchIntoCache(final ByteBuffer buf, final int position, final HllSketch sketch)
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java
index c2529ac..74be19b 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregator.java
@@ -57,7 +57,7 @@ public class DoublesSketchBuildBufferAggregator implements BufferAggregator
}
final UpdateDoublesSketch sketch = helper.getSketchAtPosition(buffer, position);
- DoublesSketches.handleMaxStreamLengthLimit(() -> sketch.update(selector.getDouble()));
+ sketch.update(selector.getDouble());
}
@Nullable
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregatorHelper.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregatorHelper.java
index 2acad90..dc7e2f1 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregatorHelper.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildBufferAggregatorHelper.java
@@ -21,6 +21,8 @@ package org.apache.druid.query.aggregation.datasketches.quantiles;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantiles.CompactDoublesSketch;
import org.apache.datasketches.quantiles.DoublesSketch;
@@ -32,6 +34,7 @@ import java.util.IdentityHashMap;
public class DoublesSketchBuildBufferAggregatorHelper
{
+ private static final MemoryRequestServer MEM_REQ_SERVER = new DefaultMemoryRequestServer();
private final int size;
private final int maxIntermediateSize;
private final IdentityHashMap<ByteBuffer, WritableMemory> memCache = new IdentityHashMap<>();
@@ -92,7 +95,8 @@ public class DoublesSketchBuildBufferAggregatorHelper
private WritableMemory getMemory(final ByteBuffer buffer)
{
- return memCache.computeIfAbsent(buffer, buf -> WritableMemory.writableWrap(buf, ByteOrder.LITTLE_ENDIAN));
+ return memCache.computeIfAbsent(buffer,
+ buf -> WritableMemory.writableWrap(buf, ByteOrder.LITTLE_ENDIAN, MEM_REQ_SERVER));
}
private void putSketch(final ByteBuffer buffer, final int position, final UpdateDoublesSketch sketch)
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildVectorAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildVectorAggregator.java
index c1074f5..af29c5b 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildVectorAggregator.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchBuildVectorAggregator.java
@@ -55,13 +55,11 @@ public class DoublesSketchBuildVectorAggregator implements VectorAggregator
final UpdateDoublesSketch sketch = helper.getSketchAtPosition(buf, position);
- DoublesSketches.handleMaxStreamLengthLimit(() -> {
- for (int i = startRow; i < endRow; i++) {
- if (nulls == null || !nulls[i]) {
- sketch.update(doubles[i]);
- }
+ for (int i = startRow; i < endRow; i++) {
+ if (nulls == null || !nulls[i]) {
+ sketch.update(doubles[i]);
}
- });
+ }
}
@Override
@@ -76,16 +74,14 @@ public class DoublesSketchBuildVectorAggregator implements VectorAggregator
final double[] doubles = selector.getDoubleVector();
final boolean[] nulls = selector.getNullVector();
- DoublesSketches.handleMaxStreamLengthLimit(() -> {
- for (int i = 0; i < numRows; i++) {
- final int idx = rows != null ? rows[i] : i;
+ for (int i = 0; i < numRows; i++) {
+ final int idx = rows != null ? rows[i] : i;
- if (nulls == null || !nulls[idx]) {
- final int position = positions[i] + positionOffset;
- helper.getSketchAtPosition(buf, position).update(doubles[idx]);
- }
+ if (nulls == null || !nulls[idx]) {
+ final int position = positions[i] + positionOffset;
+ helper.getSketchAtPosition(buf, position).update(doubles[idx]);
}
- });
+ }
}
@Override
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java
index a5f12d2..6693742 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeAggregator.java
@@ -76,12 +76,10 @@ public class DoublesSketchMergeAggregator implements Aggregator
if (object == null) {
return;
}
- DoublesSketches.handleMaxStreamLengthLimit(() -> {
- if (object instanceof DoublesSketch) {
- union.update((DoublesSketch) object);
- } else {
- union.update(selector.getDouble());
- }
- });
+ if (object instanceof DoublesSketch) {
+ union.update((DoublesSketch) object);
+ } else {
+ union.update(selector.getDouble());
+ }
}
}
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregatorHelper.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregatorHelper.java
index d9f4bad..6cb0f3a 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregatorHelper.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeBufferAggregatorHelper.java
@@ -21,6 +21,8 @@ package org.apache.druid.query.aggregation.datasketches.quantiles;
import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+import org.apache.datasketches.memory.DefaultMemoryRequestServer;
+import org.apache.datasketches.memory.MemoryRequestServer;
import org.apache.datasketches.memory.WritableMemory;
import org.apache.datasketches.quantiles.DoublesUnion;
@@ -30,6 +32,7 @@ import java.util.IdentityHashMap;
public class DoublesSketchMergeBufferAggregatorHelper
{
+ private static final MemoryRequestServer MEM_REQ_SERVER = new DefaultMemoryRequestServer();
private final int k;
private final int maxIntermediateSize;
private final IdentityHashMap<ByteBuffer, WritableMemory> memCache = new IdentityHashMap<>();
@@ -93,7 +96,8 @@ public class DoublesSketchMergeBufferAggregatorHelper
private WritableMemory getMemory(final ByteBuffer buffer)
{
- return memCache.computeIfAbsent(buffer, buf -> WritableMemory.writableWrap(buf, ByteOrder.LITTLE_ENDIAN));
+ return memCache.computeIfAbsent(buffer,
+ buf -> WritableMemory.writableWrap(buf, ByteOrder.LITTLE_ENDIAN, MEM_REQ_SERVER));
}
private void putUnion(final ByteBuffer buffer, final int position, final DoublesUnion union)
diff --git a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeVectorAggregator.java b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeVectorAggregator.java
index 92437d0..8a8e10b 100644
--- a/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeVectorAggregator.java
+++ b/extensions-core/datasketches/src/main/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchMergeVectorAggregator.java
@@ -55,14 +55,12 @@ public class DoublesSketchMergeVectorAggregator implements VectorAggregator
final DoublesUnion union = helper.getSketchAtPosition(buf, position);
- DoublesSketches.handleMaxStreamLengthLimit(() -> {
- for (int i = startRow; i < endRow; i++) {
- final DoublesSketch sketch = (DoublesSketch) vector[i];
- if (sketch != null) {
- union.update(sketch);
- }
+ for (int i = startRow; i < endRow; i++) {
+ final DoublesSketch sketch = (DoublesSketch) vector[i];
+ if (sketch != null) {
+ union.update(sketch);
}
- });
+ }
}
@Override
@@ -76,17 +74,15 @@ public class DoublesSketchMergeVectorAggregator implements VectorAggregator
{
final Object[] vector = selector.getObjectVector();
- DoublesSketches.handleMaxStreamLengthLimit(() -> {
- for (int i = 0; i < numRows; i++) {
- final DoublesSketch sketch = (DoublesSketch) vector[rows != null ? rows[i] : i];
+ for (int i = 0; i < numRows; i++) {
+ final DoublesSketch sketch = (DoublesSketch) vector[rows != null ? rows[i] : i];
- if (sketch != null) {
- final int position = positions[i] + positionOffset;
- final DoublesUnion union = helper.getSketchAtPosition(buf, position);
- union.update(sketch);
- }
+ if (sketch != null) {
+ final int position = positions[i] + positionOffset;
+ final DoublesUnion union = helper.getSketchAtPosition(buf, position);
+ union.update(sketch);
}
- });
+ }
}
@Nullable
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java
index d20f369..2c05509 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/DoublesSketchAggregatorTest.java
@@ -33,13 +33,10 @@ import org.apache.druid.query.groupby.GroupByQueryRunnerTest;
import org.apache.druid.query.groupby.ResultRow;
import org.apache.druid.query.groupby.strategy.GroupByStrategySelector;
import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.hamcrest.BaseMatcher;
-import org.hamcrest.Description;
import org.junit.After;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.ExpectedException;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -60,9 +57,6 @@ public class DoublesSketchAggregatorTest extends InitializedNullHandlingTest
@Rule
public final TemporaryFolder tempFolder = new TemporaryFolder();
- @Rule
- public final ExpectedException expectedException = ExpectedException.none();
-
public DoublesSketchAggregatorTest(final GroupByQueryConfig config, final String vectorize)
{
this.config = config;
@@ -544,12 +538,9 @@ public class DoublesSketchAggregatorTest extends InitializedNullHandlingTest
}
@Test
- public void testFailureWhenMaxStreamLengthHit() throws Exception
+ public void testSuccessWhenMaxStreamLengthHit() throws Exception
{
if (GroupByStrategySelector.STRATEGY_V1.equals(config.getDefaultStrategy())) {
- expectedException.expect(new RecursiveExceptionMatcher(IllegalStateException.class));
- expectedException.expectMessage("NullPointerException was thrown while updating Doubles sketch");
-
helper.createIndexAndRunQueryOnSegment(
new File(this.getClass().getClassLoader().getResource("quantiles/doubles_build_data.tsv").getFile()),
String.join(
@@ -633,39 +624,8 @@ public class DoublesSketchAggregatorTest extends InitializedNullHandlingTest
"}"
)
);
-
- expectedException.expect(new RecursiveExceptionMatcher(IllegalStateException.class));
- expectedException.expectMessage("NullPointerException was thrown while updating Doubles sketch");
seq.toList();
}
}
- private static class RecursiveExceptionMatcher extends BaseMatcher<Object>
- {
- private final Class<? extends Throwable> expected;
-
- private RecursiveExceptionMatcher(Class<? extends Throwable> expected)
- {
- this.expected = expected;
- }
-
- @Override
- public boolean matches(Object item)
- {
- if (expected.isInstance(item)) {
- return true;
- } else if (item instanceof Throwable) {
- if (((Throwable) item).getCause() != null) {
- return matches(((Throwable) item).getCause());
- }
- }
- return false;
- }
-
- @Override
- public void describeTo(Description description)
- {
- description.appendText("a recursive instance of ").appendText(expected.getName());
- }
- }
}
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java
index 75d556f..3df473c 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/quantiles/sql/DoublesSketchSqlAggregatorTest.java
@@ -828,14 +828,14 @@ public class DoublesSketchSqlAggregatorTest extends BaseCalciteQueryTest
}
@Test
- public void testFailWithSmallMaxStreamLength() throws Exception
+ public void testSuccessWithSmallMaxStreamLength() throws Exception
{
final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
context.put(
DoublesSketchApproxQuantileSqlAggregator.CTX_APPROX_QUANTILE_DS_MAX_STREAM_LENGTH,
1
);
- testQueryThrows(
+ testQuery(
"SELECT\n"
+ "APPROX_QUANTILE_DS(m1, 0.01),\n"
+ "APPROX_QUANTILE_DS(cnt, 0.5)\n"
@@ -856,11 +856,13 @@ public class DoublesSketchSqlAggregatorTest extends BaseCalciteQueryTest
)
.context(context)
.build()
- ),
- expectedException -> {
- expectedException.expect(IllegalStateException.class);
- expectedException.expectMessage("NullPointerException was thrown while updating Doubles sketch");
- }
+ ),
+ ImmutableList.of(
+ new Object[]{
+ 1.0,
+ 1.0
+ }
+ )
);
}
diff --git a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregationTest.java b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregationTest.java
index 7c37097..badbf0a 100644
--- a/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregationTest.java
+++ b/extensions-core/datasketches/src/test/java/org/apache/druid/query/aggregation/datasketches/tuple/ArrayOfDoublesSketchAggregationTest.java
@@ -242,6 +242,7 @@ public class ArrayOfDoublesSketchAggregationTest extends InitializedNullHandling
" \"name\": \"intersection\",",
" \"operation\": \"INTERSECT\",",
" \"nominalEntries\": 1024,",
+ " \"numberOfValues\": 2,",
" \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]",
" }},",
" {\"type\": \"arrayOfDoublesSketchToEstimate\", \"name\": \"anotb\", \"field\": {",
@@ -249,6 +250,7 @@ public class ArrayOfDoublesSketchAggregationTest extends InitializedNullHandling
" \"name\": \"anotb\",",
" \"operation\": \"NOT\",",
" \"nominalEntries\": 1024,",
+ " \"numberOfValues\": 2,",
" \"fields\": [{\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}, {\"type\": \"fieldAccess\", \"fieldName\": \"sketch\"}]",
" }},",
" {",
diff --git a/licenses.yaml b/licenses.yaml
index 5433367..260ce61 100644
--- a/licenses.yaml
+++ b/licenses.yaml
@@ -3726,7 +3726,7 @@ name: DataSketches
license_category: binary
module: java-core
license_name: Apache License version 2.0
-version: 3.0.0
+version: 3.1.0
libraries:
- org.apache.datasketches: datasketches-java
diff --git a/pom.xml b/pom.xml
index 385f45d..2823e34 100644
--- a/pom.xml
+++ b/pom.xml
@@ -83,7 +83,7 @@
<avatica.version>1.17.0</avatica.version>
<avro.version>1.9.2</avro.version>
<calcite.version>1.21.0</calcite.version>
- <datasketches.version>3.0.0</datasketches.version>
+ <datasketches.version>3.1.0</datasketches.version>
<datasketches.memory.version>2.0.0</datasketches.memory.version>
<derby.version>10.14.2.0</derby.version>
<dropwizard.metrics.version>4.0.0</dropwizard.metrics.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org