You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by jo...@apache.org on 2020/09/24 23:36:12 UTC
[druid] branch master updated (cb30b1f -> 0cc9eb4)
This is an automated email from the ASF dual-hosted git repository.
jonwei pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git.
from cb30b1f Automatically determine numShards for parallel ingestion hash partitioning (#10419)
add 0cc9eb4 Store hash partition function in dataSegment and allow segment pruning only when hash partition function is provided (#10288)
No new revisions were added by this update.
Summary of changes:
.../indexer/partitions/HashedPartitionsSpec.java | 31 ++-
.../BuildingHashBasedNumberedShardSpec.java | 17 +-
.../timeline/partition/BuildingShardSpec.java | 7 -
.../HashBasedNumberedPartialShardSpec.java | 19 +-
.../partition/HashBasedNumberedShardSpec.java | 245 ++++++++++-----------
.../timeline/partition/HashBucketShardSpec.java | 40 +++-
.../timeline/partition/HashPartitionFunction.java | 62 ++++++
.../druid/timeline/partition/HashPartitioner.java | 101 +++++++++
.../druid/timeline/partition/LinearShardSpec.java | 6 -
.../druid/timeline/partition/NoneShardSpec.java | 6 -
.../partition/NumberedOverwriteShardSpec.java | 6 -
.../timeline/partition/NumberedShardSpec.java | 6 -
.../timeline/partition/RangeBucketShardSpec.java | 13 +-
.../apache/druid/timeline/partition/ShardSpec.java | 4 -
.../partition/SingleDimensionShardSpec.java | 7 +-
.../org/apache/druid/timeline/DataSegmentTest.java | 7 -
.../BuildingHashBasedNumberedShardSpecTest.java | 22 +-
.../HashBasedNumberedPartialShardSpecTest.java | 14 +-
.../partition/HashBasedNumberedShardSpecTest.java | 245 ++++++++++++++-------
.../partition/HashBucketShardSpecTest.java | 35 ++-
.../partition/NumberedOverwriteShardSpecTest.java | 2 +-
.../timeline/partition/NumberedShardSpecTest.java | 2 +-
.../partition/PartitionHolderCompletenessTest.java | 6 +-
.../partition/SingleDimensionShardSpecTest.java | 4 +-
docs/ingestion/hadoop.md | 11 +
docs/ingestion/index.md | 2 +-
docs/ingestion/native-batch.md | 23 +-
docs/querying/query-context.md | 1 +
.../MaterializedViewSupervisorTest.java | 16 +-
indexing-hadoop/pom.xml | 5 +
.../indexer/DetermineHashedPartitionsJob.java | 13 ++
.../HadoopDruidDetermineConfigurationJob.java | 5 +
.../druid/indexer/BatchDeltaIngestionTest.java | 11 +-
.../indexer/DetermineHashedPartitionsJobTest.java | 39 +++-
.../HadoopDruidDetermineConfigurationJobTest.java | 127 +++++++++++
.../indexer/HadoopDruidIndexerConfigTest.java | 19 +-
.../druid/indexer/IndexGeneratorJobTest.java | 20 +-
.../partitions/HashedPartitionsSpecTest.java | 11 +
.../parallel/PartialDimensionCardinalityTask.java | 9 +-
.../batch/partition/HashPartitionAnalysis.java | 1 +
.../common/actions/SegmentAllocateActionTest.java | 10 +-
.../druid/indexing/common/task/IndexTaskTest.java | 78 ++++++-
.../druid/indexing/common/task/ShardSpecsTest.java | 5 +-
.../batch/parallel/GenericPartitionStatTest.java | 2 +
...ashPartitionMultiPhaseParallelIndexingTest.java | 31 ++-
.../parallel/ParallelIndexSupervisorTaskTest.java | 10 +-
.../parallel/ParallelIndexTestingFactory.java | 2 +
.../parallel/PerfectRollupWorkerTaskTest.java | 1 +
.../druid/indexing/overlord/TaskLockboxTest.java | 4 +-
.../druid/tests/hadoop/ITHadoopIndexTest.java | 2 +
.../indexer/ITPerfectRollupParallelIndexTest.java | 4 +-
.../java/org/apache/druid/query/QueryContexts.java | 6 +
.../org/apache/druid/query/QueryContextsTest.java | 24 ++
.../druid/client/CachingClusteredClient.java | 18 +-
.../druid/client/CachingClusteredClientTest.java | 238 ++++++++++++++++++--
.../IndexerSQLMetadataStorageCoordinatorTest.java | 4 +-
.../appenderator/SegmentPublisherHelperTest.java | 40 +++-
.../coordinator/duty/CompactSegmentsTest.java | 1 +
website/.spelling | 2 +
59 files changed, 1311 insertions(+), 391 deletions(-)
create mode 100644 core/src/main/java/org/apache/druid/timeline/partition/HashPartitionFunction.java
create mode 100644 core/src/main/java/org/apache/druid/timeline/partition/HashPartitioner.java
create mode 100644 indexing-hadoop/src/test/java/org/apache/druid/indexer/HadoopDruidDetermineConfigurationJobTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org