You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2024/01/09 21:05:14 UTC
(druid) branch fix_concurrent_tasks created (now fec94e10d9a)
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a change to branch fix_concurrent_tasks
in repository https://gitbox.apache.org/repos/asf/druid.git
at fec94e10d9a Update snapshots
This branch includes the following new commits:
new d4042781724 Improve handling of concurrent tasks option
new fec94e10d9a Update snapshots
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 02/02: Update snapshots
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch fix_concurrent_tasks
in repository https://gitbox.apache.org/repos/asf/druid.git
commit fec94e10d9a7ffcd4b771e1d4691c2a8bc862636
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Tue Jan 9 13:04:58 2024 -0800
Update snapshots
---
.../compaction-config-dialog.spec.tsx.snap | 2640 ++++++++++----------
1 file changed, 1312 insertions(+), 1328 deletions(-)
diff --git a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
index 11949933699..7bb67d84f97 100644
--- a/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
+++ b/web-console/src/dialogs/compaction-config-dialog/__snapshots__/compaction-config-dialog.spec.tsx.snap
@@ -15,351 +15,347 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (dynamic
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "dynamic",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "dynamic",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -408,351 +404,347 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (hashed p
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "hashed",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "hashed",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -801,351 +793,347 @@ exports[`CompactionConfigDialog matches snapshot with compactionConfig (range pa
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "range",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "range",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
@@ -1194,351 +1182,347 @@ exports[`CompactionConfigDialog matches snapshot without compactionConfig 1`] =
<div
className="content"
>
- <AutoForm
- fields={
- Array [
- Object {
- "defaultValue": "P1D",
- "info": <p>
- The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
- </p>,
- "name": "skipOffsetFromLatest",
- "suggestions": Array [
- "PT0H",
- "PT1H",
- "P1D",
- "P3D",
- ],
- "type": "string",
- },
- Object {
- "info": <p>
- For perfect rollup, you should use either
- <Unknown>
- hashed
- </Unknown>
- (partitioning based on the hash of dimensions in each row) or
- <Unknown>
- range
- </Unknown>
- (based on several dimensions). For best-effort rollup, you should use
- <Unknown>
- dynamic
- </Unknown>
- .
- </p>,
- "label": "Partitioning type",
- "name": "tuningConfig.partitionsSpec.type",
- "suggestions": Array [
- "dynamic",
- "hashed",
- "range",
- ],
- "type": "string",
- },
- Object {
- "defaultValue": 5000000,
- "defined": [Function],
- "info": <React.Fragment>
- Determines how many rows are in each segment.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- },
- Object {
- "defaultValue": 20000000,
- "defined": [Function],
- "info": <React.Fragment>
- Total number of rows in segments waiting for being pushed.
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxTotalRows",
- "type": "number",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
- </p>
- <p>
- A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
- </p>
- <p>
- If
- <Unknown>
- numShards
- </Unknown>
- is left unspecified, the Parallel task will determine
-
- <Unknown>
- numShards
- </Unknown>
- automatically by
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- .
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "placeholder": "(defaults to 500000)",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- renamed to
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- </p>
- <p>
- If
+ <React.Fragment>
+ <AutoForm
+ fields={
+ Array [
+ Object {
+ "defaultValue": "P1D",
+ "info": <p>
+ The offset for searching segments to be compacted. Strongly recommended to set for realtime dataSources.
+ </p>,
+ "name": "skipOffsetFromLatest",
+ "suggestions": Array [
+ "PT0H",
+ "PT1H",
+ "P1D",
+ "P3D",
+ ],
+ "type": "string",
+ },
+ Object {
+ "info": <p>
+ For perfect rollup, you should use either
<Unknown>
- numShards
+ hashed
</Unknown>
- is left unspecified, the Parallel task will determine
-
+ (partitioning based on the hash of dimensions in each row) or
<Unknown>
- numShards
+ range
</Unknown>
- automatically by
+ (based on several dimensions). For best-effort rollup, you should use
<Unknown>
- targetRowsPerSegment
+ dynamic
</Unknown>
.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
- </p>
- <p>
- Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- numShards
- </Unknown>
- will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.numShards",
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on. Leave blank to select all dimensions.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "placeholder": "(all dimensions)",
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimension to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimension",
- "required": true,
- "type": "string",
- },
- Object {
- "defined": [Function],
- "info": <p>
- The dimensions to partition on.
- </p>,
- "name": "tuningConfig.partitionsSpec.partitionDimensions",
- "required": true,
- "type": "string-array",
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
- </p>
- <p>
- Note that either
- <Unknown>
- targetRowsPerSegment
- </Unknown>
- or
- <Unknown>
- maxRowsPerSegment
- </Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defined": [Function],
- "info": <React.Fragment>
- <p>
- Maximum number of rows to include in a partition.
- </p>
- <p>
- Note that either
+ </p>,
+ "label": "Partitioning type",
+ "name": "tuningConfig.partitionsSpec.type",
+ "suggestions": Array [
+ "dynamic",
+ "hashed",
+ "range",
+ ],
+ "type": "string",
+ },
+ Object {
+ "defaultValue": 5000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Determines how many rows are in each segment.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 20000000,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Total number of rows in segments waiting for being pushed.
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxTotalRows",
+ "type": "number",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If the segments generated are a sub-optimal size for the requested partition dimensions, consider setting this field.
+ </p>
+ <p>
+ A target row count for each partition. Each partition will have a row count close to the target assuming evenly distributed keys. Defaults to 5 million if numShards is null.
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "placeholder": "(defaults to 500000)",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ renamed to
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ </p>
+ <p>
+ If
+ <Unknown>
+ numShards
+ </Unknown>
+ is left unspecified, the Parallel task will determine
+
+ <Unknown>
+ numShards
+ </Unknown>
+ automatically by
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ .
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ If you know the optimal number of shards and want to speed up the time it takes for compaction to run, set this field.
+ </p>
+ <p>
+ Directly specify the number of shards to create. If this is specified and 'intervals' is specified in the granularitySpec, the index task can skip the determine intervals/partitions pass through the data.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ numShards
+ </Unknown>
+ will be used to evenly distribute rows across partitions and find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.numShards",
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on. Leave blank to select all dimensions.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "placeholder": "(all dimensions)",
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimension to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimension",
+ "required": true,
+ "type": "string",
+ },
+ Object {
+ "defined": [Function],
+ "info": <p>
+ The dimensions to partition on.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.partitionDimensions",
+ "required": true,
+ "type": "string-array",
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Target number of rows to include in a partition, should be a number that targets segments of 500MB~1GB.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.targetRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defined": [Function],
+ "info": <React.Fragment>
+ <p>
+ Maximum number of rows to include in a partition.
+ </p>
+ <p>
+ Note that either
+ <Unknown>
+ targetRowsPerSegment
+ </Unknown>
+ or
+ <Unknown>
+ maxRowsPerSegment
+ </Unknown>
+ will be used to find the best partitioning. Leave blank to show all properties.
+ </p>
+ </React.Fragment>,
+ "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
+ "required": [Function],
+ "type": "number",
+ "zeroMeansUndefined": true,
+ },
+ Object {
+ "defaultValue": false,
+ "defined": [Function],
+ "info": <p>
+ Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
+ </p>,
+ "name": "tuningConfig.partitionsSpec.assumeGrouped",
+ "type": "boolean",
+ },
+ Object {
+ "defaultValue": 1,
+ "info": <React.Fragment>
+ Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.maxNumConcurrentSubTasks",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": -1,
+ "info": <React.Fragment>
+ <p>
+ Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
+ </p>
+ <p>
+ Default: -1 (unlimited)
+ </p>
+ </React.Fragment>,
+ "min": -1,
+ "name": "tuningConfig.maxColumnsToMerge",
+ "type": "number",
+ },
+ Object {
+ "defaultValue": 10,
+ "defined": [Function],
+ "info": <React.Fragment>
+ Maximum number of merge tasks which can be run at the same time.
+ </React.Fragment>,
+ "min": 1,
+ "name": "tuningConfig.totalNumMergeTasks",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1073741824,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
+ </React.Fragment>,
+ "min": 1000000,
+ "name": "tuningConfig.splitHintSpec.maxSplitSize",
+ "type": "number",
+ },
+ Object {
+ "adjustment": [Function],
+ "defaultValue": 1000,
+ "hideInMore": true,
+ "info": <React.Fragment>
+ Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
<Unknown>
- targetRowsPerSegment
+ jute.maxbuffer
</Unknown>
- or
+ ) and the max packet size in MySQL (
<Unknown>
- maxRowsPerSegment
+ max_allowed_packet
</Unknown>
- will be used to find the best partitioning. Leave blank to show all properties.
- </p>
- </React.Fragment>,
- "name": "tuningConfig.partitionsSpec.maxRowsPerSegment",
- "required": [Function],
- "type": "number",
- "zeroMeansUndefined": true,
- },
- Object {
- "defaultValue": false,
- "defined": [Function],
- "info": <p>
- Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.
- </p>,
- "name": "tuningConfig.partitionsSpec.assumeGrouped",
- "type": "boolean",
- },
- Object {
- "defaultValue": 1,
- "info": <React.Fragment>
- Maximum number of tasks which can be run at the same time. The supervisor task would spawn worker tasks up to maxNumConcurrentSubTasks regardless of the available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.maxNumConcurrentSubTasks",
- "type": "number",
- },
- Object {
- "defaultValue": -1,
- "info": <React.Fragment>
- <p>
- Limit of the number of segments to merge in a single phase when merging segments for publishing. This limit affects the total number of columns present in a set of segments to merge. If the limit is exceeded, segment merging occurs in multiple phases. Druid merges at least 2 segments per phase, regardless of this setting.
- </p>
- <p>
- Default: -1 (unlimited)
- </p>
- </React.Fragment>,
- "min": -1,
- "name": "tuningConfig.maxColumnsToMerge",
- "type": "number",
- },
- Object {
- "defaultValue": 10,
- "defined": [Function],
- "info": <React.Fragment>
- Maximum number of merge tasks which can be run at the same time.
- </React.Fragment>,
- "min": 1,
- "name": "tuningConfig.totalNumMergeTasks",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1073741824,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of bytes of input segments to process in a single task. If a single segment is larger than this number, it will be processed by itself in a single task (input segments are never split across tasks).
- </React.Fragment>,
- "min": 1000000,
- "name": "tuningConfig.splitHintSpec.maxSplitSize",
- "type": "number",
- },
- Object {
- "adjustment": [Function],
- "defaultValue": 1000,
- "hideInMore": true,
- "info": <React.Fragment>
- Maximum number of input segments to process in a single subtask. This limit is to avoid task failures when the ingestion spec is too long. There are two known limits on the max size of serialized ingestion spec, i.e., the max ZNode size in ZooKeeper (
- <Unknown>
- jute.maxbuffer
- </Unknown>
- ) and the max packet size in MySQL (
- <Unknown>
- max_allowed_packet
- </Unknown>
- ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
- </React.Fragment>,
- "label": "Max num files (segments)",
- "min": 1,
- "name": "tuningConfig.splitHintSpec.maxNumFiles",
- "type": "number",
- },
+ ). These can make ingestion tasks fail if the serialized ingestion spec size hits one of them.
+ </React.Fragment>,
+ "label": "Max num files (segments)",
+ "min": 1,
+ "name": "tuningConfig.splitHintSpec.maxNumFiles",
+ "type": "number",
+ },
+ ]
+ }
+ model={
Object {
- "adjustValue": [Function],
- "defaultValue": undefined,
- "info": <p>
- Allows or forbids concurrent compactions.
- </p>,
- "label": "Allow concurrent compactions (experimental)",
- "name": "taskContext.taskLockType",
- "type": "boolean",
- "valueAdjustment": [Function],
- },
- ]
- }
- model={
- Object {
- "dataSource": "test1",
- "tuningConfig": Object {
- "partitionsSpec": Object {
- "type": "dynamic",
- },
- },
+ "dataSource": "test1",
+ "tuningConfig": Object {
+ "partitionsSpec": Object {
+ "type": "dynamic",
+ },
+ },
+ }
}
- }
- onChange={[Function]}
- />
+ onChange={[Function]}
+ />
+ <Blueprint4.Switch
+ checked={false}
+ label="Allow concurrent compactions (experimental)"
+ onChange={[Function]}
+ />
+ </React.Fragment>
</div>
<div
className="bp4-dialog-footer"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org
(druid) 01/02: Improve handling of concurrent tasks option
Posted by vo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch fix_concurrent_tasks
in repository https://gitbox.apache.org/repos/asf/druid.git
commit d40427817243eb402532df130e2293f33b5e0701
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Tue Jan 9 12:42:38 2024 -0800
Improve handling of concurrent tasks option
---
web-console/src/components/auto-form/auto-form.tsx | 10 +----
.../compaction-config-dialog.tsx | 27 +++++++++----
.../compaction-config/compaction-config.tsx | 9 -----
.../druid-models/ingestion-spec/ingestion-spec.tsx | 5 ++-
.../src/views/load-data-view/load-data-view.tsx | 47 +++++++++-------------
5 files changed, 44 insertions(+), 54 deletions(-)
diff --git a/web-console/src/components/auto-form/auto-form.tsx b/web-console/src/components/auto-form/auto-form.tsx
index c63f7a7bc50..357046876af 100644
--- a/web-console/src/components/auto-form/auto-form.tsx
+++ b/web-console/src/components/auto-form/auto-form.tsx
@@ -72,10 +72,6 @@ export interface Field<M> {
hide?: Functor<M, boolean>;
hideInMore?: Functor<M, boolean>;
valueAdjustment?: (value: any) => any;
- /**
- * An optional callback to transform the value before it is set on the input
- */
- adjustValue?: (value: any) => any;
adjustment?: (model: Partial<M>, oldModel: Partial<M>) => Partial<M>;
issueWithValue?: (value: any) => string | undefined;
@@ -382,14 +378,12 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
const disabled = AutoForm.evaluateFunctor(field.disabled, model, false);
const intent = required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined;
- const adjustedValue = field.adjustValue ? field.adjustValue(shownValue) : shownValue;
-
return (
<ButtonGroup large={large}>
<Button
intent={intent}
disabled={disabled}
- active={adjustedValue === false}
+ active={shownValue === false}
onClick={() => {
this.fieldChange(field, false);
if (onFinalize) onFinalize();
@@ -400,7 +394,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
<Button
intent={intent}
disabled={disabled}
- active={adjustedValue === true}
+ active={shownValue === true}
onClick={() => {
this.fieldChange(field, true);
if (onFinalize) onFinalize();
diff --git a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
index 8a3a232072e..ec2135e43aa 100644
--- a/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
+++ b/web-console/src/dialogs/compaction-config-dialog/compaction-config-dialog.tsx
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-import { Button, Callout, Classes, Code, Dialog, Intent } from '@blueprintjs/core';
+import { Button, Callout, Classes, Code, Dialog, Intent, Switch } from '@blueprintjs/core';
import React, { useState } from 'react';
import type { FormJsonTabs } from '../../components';
@@ -26,7 +26,7 @@ import {
COMPACTION_CONFIG_FIELDS,
compactionConfigHasLegacyInputSegmentSizeBytesSet,
} from '../../druid-models';
-import { deepDelete, formatBytesCompact } from '../../utils';
+import { deepDelete, deepGet, deepSet, formatBytesCompact } from '../../utils';
import { CompactionHistoryDialog } from '../compaction-history-dialog/compaction-history-dialog';
import './compaction-config-dialog.scss';
@@ -96,11 +96,24 @@ export const CompactionConfigDialog = React.memo(function CompactionConfigDialog
/>
<div className="content">
{currentTab === 'form' ? (
- <AutoForm
- fields={COMPACTION_CONFIG_FIELDS}
- model={currentConfig}
- onChange={m => setCurrentConfig(m as CompactionConfig)}
- />
+ <>
+ <AutoForm
+ fields={COMPACTION_CONFIG_FIELDS}
+ model={currentConfig}
+ onChange={m => setCurrentConfig(m as CompactionConfig)}
+ />
+ <Switch
+ label="Allow concurrent compactions (experimental)"
+ checked={typeof deepGet(currentConfig, 'taskContext.taskLockType') === 'string'}
+ onChange={() => {
+ setCurrentConfig(
+ (typeof deepGet(currentConfig, 'taskContext.taskLockType') === 'string'
+ ? deepDelete(currentConfig, 'taskContext.taskLockType')
+ : deepSet(currentConfig, 'taskContext.taskLockType', 'REPLACE')) as any,
+ );
+ }}
+ />
+ </>
) : (
<JsonInput
value={currentConfig}
diff --git a/web-console/src/druid-models/compaction-config/compaction-config.tsx b/web-console/src/druid-models/compaction-config/compaction-config.tsx
index a074af7ae14..8c03b6a5786 100644
--- a/web-console/src/druid-models/compaction-config/compaction-config.tsx
+++ b/web-console/src/druid-models/compaction-config/compaction-config.tsx
@@ -354,13 +354,4 @@ export const COMPACTION_CONFIG_FIELDS: Field<CompactionConfig>[] = [
</>
),
},
- {
- name: 'taskContext.taskLockType',
- type: 'boolean',
- label: 'Allow concurrent compactions (experimental)',
- defaultValue: undefined,
- valueAdjustment: v => (v ? 'REPLACE' : undefined),
- adjustValue: v => v === 'REPLACE',
- info: <p>Allows or forbids concurrent compactions.</p>,
- },
];
diff --git a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
index 7dd6bd4e82f..3594f70c992 100644
--- a/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
+++ b/web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx
@@ -347,7 +347,10 @@ export function normalizeSpec(spec: Partial<IngestionSpec>): IngestionSpec {
spec = deepSetIfUnset(spec, 'spec.tuningConfig.type', specType);
if (spec.context?.taskLockType !== undefined) {
- spec.context.taskLockType = spec.spec?.ioConfig.appendToExisting ? 'APPEND' : 'REPLACE';
+ spec.context.taskLockType =
+ isStreamingSpec(spec) || deepGet(spec, 'spec.ioConfig.appendToExisting')
+ ? 'APPEND'
+ : 'REPLACE';
}
return spec as IngestionSpec;
diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
index af39c2d0385..db7e8e0f093 100644
--- a/web-console/src/views/load-data-view/load-data-view.tsx
+++ b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -3134,8 +3134,6 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
const { spec } = this.state;
const parallel = deepGet(spec, 'spec.tuningConfig.maxNumConcurrentSubTasks') > 1;
- const appendToExisting = spec.spec?.ioConfig.appendToExisting;
-
return (
<>
<div className="main">
@@ -3161,6 +3159,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
name: 'spec.ioConfig.appendToExisting',
label: 'Append to existing',
type: 'boolean',
+ defined: s => !isStreamingSpec(s),
defaultValue: false,
// appendToExisting can only be set on 'dynamic' portioning.
// We chose to show it always and instead have a specific message, separate from this form, to notify the user of the issue.
@@ -3171,37 +3170,27 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
</>
),
},
- {
- name: 'context.taskLockType',
- type: 'boolean',
- label: `Allow concurrent ${
- appendToExisting ? 'append' : 'replace'
- } tasks (experimental)`,
- defaultValue: undefined,
- valueAdjustment: v => {
- if (!v) return undefined;
-
- if (isStreamingSpec(spec)) {
- return 'APPEND';
- } else {
- return appendToExisting ? 'APPEND' : 'REPLACE';
- }
- },
- adjustValue: v => {
- if (v === undefined) return false;
-
- if (isStreamingSpec(spec)) {
- return v === 'APPEND';
- }
-
- return v === (appendToExisting ? 'APPEND' : 'REPLACE');
- },
- info: <p>Allows or forbids concurrent tasks.</p>,
- },
]}
model={spec}
onChange={this.updateSpec}
/>
+ <Switch
+ label="Allow concurrent tasks (experimental)"
+ checked={typeof deepGet(spec, 'context.taskLockType') === 'string'}
+ onChange={() => {
+ this.updateSpec(
+ typeof deepGet(spec, 'context.taskLockType') === 'string'
+ ? deepDelete(spec, 'context.taskLockType')
+ : deepSet(
+ spec,
+ 'context.taskLockType',
+ isStreamingSpec(spec) || deepGet(spec, 'spec.ioConfig.appendToExisting')
+ ? 'APPEND'
+ : 'REPLACE',
+ ),
+ );
+ }}
+ />
</div>
<div className="other">
<H5>Parse error reporting</H5>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org