You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by GitBox <gi...@apache.org> on 2018/12/07 04:05:29 UTC
[GitHub] asdf2014 closed pull request #6713: Fix travis timeout in
BufferHashGrouperTest
asdf2014 closed pull request #6713: Fix travis timeout in BufferHashGrouperTest
URL: https://github.com/apache/incubator-druid/pull/6713
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/processing/pom.xml b/processing/pom.xml
index 4963e32acbe..35f6dd0a52d 100644
--- a/processing/pom.xml
+++ b/processing/pom.xml
@@ -195,7 +195,7 @@
<!-- set default options -->
<argLine>
-Xmx512m
- -XX:MaxDirectMemorySize=1500m
+ -XX:MaxDirectMemorySize=2500m
-Duser.language=en
-Duser.GroupByQueryRunnerTest.javacountry=US
-Dfile.encoding=UTF-8
diff --git a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
index c28657a1c55..64b270d0f4d 100644
--- a/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
+++ b/processing/src/test/java/org/apache/druid/query/groupby/epinephelinae/BufferHashGrouperTest.java
@@ -20,16 +20,13 @@
package org.apache.druid.query.groupby.epinephelinae;
import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
import com.google.common.collect.Ordering;
-import com.google.common.io.Files;
import com.google.common.primitives.Ints;
import org.apache.druid.common.config.NullHandling;
import org.apache.druid.data.input.MapBasedRow;
-import org.apache.druid.java.util.common.ByteBufferUtils;
import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.query.aggregation.CountAggregatorFactory;
import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
@@ -39,12 +36,7 @@
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@@ -112,7 +104,7 @@ public int compare(Grouper.Entry<Integer> o1, Grouper.Entry<Integer> o2)
public void testGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, 2);
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, 2, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 219 : 210;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
@@ -137,38 +129,29 @@ public void testGrowing()
}
@Test
- public void testGrowing2()
+ public void testGrowingOverflowingInteger()
{
- final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 2_000_000_000, 2);
- final int expectedMaxSize = NullHandling.replaceWithDefault() ? 40988516 : 39141224;
-
- columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
- for (int i = 0; i < expectedMaxSize; i++) {
- Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
- }
- Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
- }
-
- @Test
- public void testGrowing3()
- {
- final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, Integer.MAX_VALUE, 2);
- final int expectedMaxSize = NullHandling.replaceWithDefault() ? 44938972 : 42955456;
-
- columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
- for (int i = 0; i < expectedMaxSize; i++) {
- Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
+ // This test checks the bug reported in https://github.com/apache/incubator-druid/pull/4333 only when
+ // NullHandling.replaceWithDefault() is true
+ if (NullHandling.replaceWithDefault()) {
+ final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
+ // the buffer size below is chosen to test integer overflow in ByteBufferHashTable.adjustTableWhenFull().
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 1_900_000_000, 2, 0.3f);
+ final int expectedMaxSize = 15323979;
+
+ columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
+ for (int i = 0; i < expectedMaxSize; i++) {
+ Assert.assertTrue(String.valueOf(i), grouper.aggregate(i).isOk());
+ }
+ Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
- Assert.assertFalse(grouper.aggregate(expectedMaxSize).isOk());
}
@Test
public void testNoGrowing()
{
final TestColumnSelectorFactory columnSelectorFactory = GrouperTestUtil.newColumnSelectorFactory();
- final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE);
+ final Grouper<Integer> grouper = makeGrouper(columnSelectorFactory, 10000, Integer.MAX_VALUE, 0.75f);
final int expectedMaxSize = NullHandling.replaceWithDefault() ? 267 : 258;
columnSelectorFactory.setRow(new MapBasedRow(0, ImmutableMap.of("value", 10L)));
@@ -195,22 +178,11 @@ public void testNoGrowing()
private BufferHashGrouper<Integer> makeGrouper(
TestColumnSelectorFactory columnSelectorFactory,
int bufferSize,
- int initialBuckets
+ int initialBuckets,
+ float maxLoadFactor
)
{
- final MappedByteBuffer buffer;
-
- try {
- final File file = temporaryFolder.newFile();
- try (final FileChannel channel = new FileOutputStream(file).getChannel()) {
- channel.truncate(bufferSize);
- }
- buffer = Files.map(file, FileChannel.MapMode.READ_WRITE, bufferSize);
- closerRule.closeLater(() -> ByteBufferUtils.unmap(buffer));
- }
- catch (IOException e) {
- throw Throwables.propagate(e);
- }
+ final ByteBuffer buffer = ByteBuffer.allocateDirect(bufferSize);
final BufferHashGrouper<Integer> grouper = new BufferHashGrouper<>(
Suppliers.ofInstance(buffer),
@@ -221,7 +193,7 @@ public void testNoGrowing()
new CountAggregatorFactory("count")
},
Integer.MAX_VALUE,
- 0.75f,
+ maxLoadFactor,
initialBuckets,
true
);
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org