You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by jo...@apache.org on 2019/08/13 19:10:37 UTC
[incubator-druid] branch master updated: Add missing
maxBytesInMemory in tuningConfig for auto compaction (#8274)
This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/master by this push:
new a5c9c29 Add missing maxBytesInMemory in tuningConfig for auto compaction (#8274)
a5c9c29 is described below
commit a5c9c2950fdfb1012eb1e758eda31e15f040e6ae
Author: Jihoon Son <ji...@apache.org>
AuthorDate: Tue Aug 13 12:10:26 2019 -0700
Add missing maxBytesInMemory in tuningConfig for auto compaction (#8274)
* Add missing tuningConfigs for auto compaciton
* Add doc
* add test
---
docs/content/configuration/index.md | 1 +
.../indexing/ClientCompactQueryTuningConfig.java | 36 +++++++++++++++----
.../coordinator/DataSourceCompactionConfig.java | 13 +++++--
.../DataSourceCompactionConfigTest.java | 41 ++++++++++++++++++----
4 files changed, 77 insertions(+), 14 deletions(-)
diff --git a/docs/content/configuration/index.md b/docs/content/configuration/index.md
index fcccac7..134ff18 100644
--- a/docs/content/configuration/index.md
+++ b/docs/content/configuration/index.md
@@ -876,6 +876,7 @@ If you see this problem, it's recommended to set `skipOffsetFromLatest` to some
|Property|Description|Required|
|--------|-----------|--------|
|`maxRowsInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 1000000)|
+|`maxBytesInMemory`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (1/6 of max JVM memory)|
|`maxTotalRows`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 20000000)|
|`indexSpec`|See [IndexSpec](../ingestion/native_tasks.html#indexspec)|no|
|`maxPendingPersists`|See [tuningConfig for indexTask](../ingestion/native_tasks.html#tuningconfig)|no (default = 0 (meaning one persist can be running concurrently with ingestion, and none can be queued up))|
diff --git a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java
index 14cf45b..343bf49 100644
--- a/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java
+++ b/server/src/main/java/org/apache/druid/client/indexing/ClientCompactQueryTuningConfig.java
@@ -32,9 +32,11 @@ public class ClientCompactQueryTuningConfig
@Nullable
private final Integer maxRowsPerSegment;
@Nullable
+ private final Long maxBytesInMemory;
+ @Nullable
private final Integer maxRowsInMemory;
@Nullable
- private final Integer maxTotalRows;
+ private final Long maxTotalRows;
@Nullable
private final IndexSpec indexSpec;
@Nullable
@@ -50,9 +52,12 @@ public class ClientCompactQueryTuningConfig
return new ClientCompactQueryTuningConfig(
maxRowsPerSegment,
userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxRowsInMemory(),
+ userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxBytesInMemory(),
userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxTotalRows(),
userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getIndexSpec(),
- userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(),
+ userCompactionTaskQueryTuningConfig == null
+ ? null
+ : userCompactionTaskQueryTuningConfig.getMaxPendingPersists(),
userCompactionTaskQueryTuningConfig == null ? null : userCompactionTaskQueryTuningConfig.getPushTimeout()
);
}
@@ -61,13 +66,15 @@ public class ClientCompactQueryTuningConfig
public ClientCompactQueryTuningConfig(
@JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment,
@JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory,
- @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows,
+ @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory,
+ @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows,
@JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
@JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
@JsonProperty("pushTimeout") @Nullable Long pushTimeout
)
{
this.maxRowsPerSegment = maxRowsPerSegment;
+ this.maxBytesInMemory = maxBytesInMemory;
this.maxRowsInMemory = maxRowsInMemory;
this.maxTotalRows = maxTotalRows;
this.indexSpec = indexSpec;
@@ -97,7 +104,14 @@ public class ClientCompactQueryTuningConfig
@JsonProperty
@Nullable
- public Integer getMaxTotalRows()
+ public Long getMaxBytesInMemory()
+ {
+ return maxBytesInMemory;
+ }
+
+ @JsonProperty
+ @Nullable
+ public Long getMaxTotalRows()
{
return maxTotalRows;
}
@@ -134,6 +148,7 @@ public class ClientCompactQueryTuningConfig
}
ClientCompactQueryTuningConfig that = (ClientCompactQueryTuningConfig) o;
return Objects.equals(maxRowsPerSegment, that.maxRowsPerSegment) &&
+ Objects.equals(maxBytesInMemory, that.maxBytesInMemory) &&
Objects.equals(maxRowsInMemory, that.maxRowsInMemory) &&
Objects.equals(maxTotalRows, that.maxTotalRows) &&
Objects.equals(indexSpec, that.indexSpec) &&
@@ -144,14 +159,23 @@ public class ClientCompactQueryTuningConfig
@Override
public int hashCode()
{
- return Objects.hash(maxRowsPerSegment, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout);
+ return Objects.hash(
+ maxRowsPerSegment,
+ maxBytesInMemory,
+ maxRowsInMemory,
+ maxTotalRows,
+ indexSpec,
+ maxPendingPersists,
+ pushTimeout
+ );
}
@Override
public String toString()
{
- return getClass().getSimpleName() + "{" +
+ return "ClientCompactQueryTuningConfig{" +
"maxRowsPerSegment=" + maxRowsPerSegment +
+ ", maxBytesInMemory=" + maxBytesInMemory +
", maxRowsInMemory=" + maxRowsInMemory +
", maxTotalRows=" + maxTotalRows +
", indexSpec=" + indexSpec +
diff --git a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
index 09f4e81..ebf4da6 100644
--- a/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
+++ b/server/src/main/java/org/apache/druid/server/coordinator/DataSourceCompactionConfig.java
@@ -237,13 +237,22 @@ public class DataSourceCompactionConfig
@JsonCreator
public UserCompactTuningConfig(
@JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory,
- @JsonProperty("maxTotalRows") @Nullable Integer maxTotalRows,
+ @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory,
+ @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows,
@JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
@JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
@JsonProperty("pushTimeout") @Nullable Long pushTimeout
)
{
- super(null, maxRowsInMemory, maxTotalRows, indexSpec, maxPendingPersists, pushTimeout);
+ super(
+ null,
+ maxRowsInMemory,
+ maxBytesInMemory,
+ maxTotalRows,
+ indexSpec,
+ maxPendingPersists,
+ pushTimeout
+ );
}
@Override
diff --git a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
index 9b8dbbd..1f76769 100644
--- a/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
+++ b/server/src/test/java/org/apache/druid/server/coordinator/DataSourceCompactionConfigTest.java
@@ -22,6 +22,10 @@ package org.apache.druid.server.coordinator;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import org.apache.druid.jackson.DefaultObjectMapper;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.data.CompressionFactory.LongEncodingStrategy;
+import org.apache.druid.segment.data.CompressionStrategy;
+import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
import org.apache.druid.server.coordinator.DataSourceCompactionConfig.UserCompactTuningConfig;
import org.joda.time.Period;
import org.junit.Assert;
@@ -97,7 +101,7 @@ public class DataSourceCompactionConfigTest
@Test
public void testSerdeUserCompactTuningConfig() throws IOException
{
- final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null);
+ final UserCompactTuningConfig config = new UserCompactTuningConfig(null, null, null, null, null, null);
final String json = objectMapper.writeValueAsString(config);
// Check maxRowsPerSegment doesn't exist in the JSON string
Assert.assertFalse(json.contains("maxRowsPerSegment"));
@@ -118,7 +122,8 @@ public class DataSourceCompactionConfigTest
new Period(3600),
new UserCompactTuningConfig(
null,
- 10000,
+ null,
+ 10000L,
null,
null,
null
@@ -140,7 +145,7 @@ public class DataSourceCompactionConfigTest
}
@Test
- public void testTargetCompactionSizeBytesWithMaxRowsPerSegment()
+ public void testSerdeTargetCompactionSizeBytesWithMaxRowsPerSegment()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage(
@@ -160,7 +165,7 @@ public class DataSourceCompactionConfigTest
}
@Test
- public void testTargetCompactionSizeBytesWithMaxTotalRows()
+ public void testSerdeTargetCompactionSizeBytesWithMaxTotalRows()
{
expectedException.expect(IllegalArgumentException.class);
expectedException.expectMessage(
@@ -176,7 +181,8 @@ public class DataSourceCompactionConfigTest
new Period(3600),
new UserCompactTuningConfig(
null,
- 10000,
+ null,
+ 10000L,
null,
null,
null
@@ -198,7 +204,8 @@ public class DataSourceCompactionConfigTest
new Period(3600),
new UserCompactTuningConfig(
null,
- 10000,
+ null,
+ 10000L,
null,
null,
null
@@ -219,4 +226,26 @@ public class DataSourceCompactionConfigTest
Assert.assertEquals(config.getTuningConfig(), fromJson.getTuningConfig());
Assert.assertEquals(config.getTaskContext(), fromJson.getTaskContext());
}
+
+ @Test
+ public void testSerdeUserCompactionTuningConfig() throws IOException
+ {
+ final UserCompactTuningConfig tuningConfig = new UserCompactTuningConfig(
+ 1000,
+ 10000L,
+ 2000L,
+ new IndexSpec(
+ new RoaringBitmapSerdeFactory(false),
+ CompressionStrategy.LZF,
+ CompressionStrategy.UNCOMPRESSED,
+ LongEncodingStrategy.LONGS
+ ),
+ 1,
+ 3000L
+ );
+
+ final String json = objectMapper.writeValueAsString(tuningConfig);
+ final UserCompactTuningConfig fromJson = objectMapper.readValue(json, UserCompactTuningConfig.class);
+ Assert.assertEquals(tuningConfig, fromJson);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org