You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by su...@apache.org on 2021/04/09 04:03:31 UTC
[druid] branch master updated (338886f -> 8264203)
This is an automated email from the ASF dual-hosted git repository.
suneet pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git.
from 338886f vector group by support for string expressions (#11010)
add 8264203 Allow client to configure batch ingestion task to wait to complete until segments are confirmed to be available by other (#10676)
No new revisions were added by this update.
Summary of changes:
.../org/apache/druid/indexer/IngestionState.java | 1 +
docs/ingestion/compaction.md | 2 +-
docs/ingestion/hadoop.md | 1 +
docs/ingestion/native-batch.md | 2 +
.../MaterializedViewSupervisorSpec.java | 3 +-
.../druid/indexing/kafka/KafkaIndexTaskTest.java | 4 +-
.../indexing/kinesis/KinesisIndexTaskTest.java | 4 +-
.../apache/druid/indexer/HadoopTuningConfig.java | 26 ++-
.../druid/indexer/BatchDeltaIngestionTest.java | 1 +
.../indexer/DetermineHashedPartitionsJobTest.java | 1 +
.../druid/indexer/DeterminePartitionsJobTest.java | 1 +
.../indexer/HadoopDruidIndexerConfigTest.java | 1 +
.../druid/indexer/HadoopTuningConfigTest.java | 2 +
.../druid/indexer/IndexGeneratorJobTest.java | 1 +
.../org/apache/druid/indexer/JobHelperTest.java | 1 +
.../indexer/path/GranularityPathSpecTest.java | 1 +
.../guice/IndexingServiceTuningConfigModule.java | 4 +-
.../IngestionStatsAndErrorsTaskReportData.java | 25 ++-
.../apache/druid/indexing/common/TaskToolbox.java | 2 +-
.../druid/indexing/common/TaskToolboxFactory.java | 2 +-
.../common/task/AbstractBatchIndexTask.java | 65 ++++++
.../task/AppenderatorDriverRealtimeIndexTask.java | 11 +-
.../druid/indexing/common/task/CompactionTask.java | 228 +++++++++++++++++++--
.../indexing/common/task/HadoopIndexTask.java | 19 +-
.../druid/indexing/common/task/IndexTask.java | 51 ++++-
.../parallel/ParallelIndexSupervisorTask.java | 83 +++++++-
.../batch/parallel/ParallelIndexTuningConfig.java | 10 +-
.../SeekableStreamIndexTaskRunner.java | 15 +-
.../druid/indexing/common/TaskToolboxTest.java | 2 +-
.../AppenderatorDriverRealtimeIndexTaskTest.java | 4 +-
.../task/ClientCompactionTaskQuerySerdeTest.java | 1 +
.../common/task/CompactionTaskParallelRunTest.java | 1 +
.../common/task/CompactionTaskRunTest.java | 1 +
.../indexing/common/task/CompactionTaskTest.java | 181 +++++++++++++++-
.../common/task/CompactionTuningConfigTest.java | 202 ++++++++++++++++++
.../indexing/common/task/IndexTaskSerdeTest.java | 14 +-
.../druid/indexing/common/task/IndexTaskTest.java | 190 +++++++++++++++++
.../common/task/RealtimeIndexTaskTest.java | 4 +-
.../indexing/common/task/TaskReportSerdeTest.java | 5 +-
.../druid/indexing/common/task/TaskSerdeTest.java | 5 +-
.../AbstractParallelIndexSupervisorTaskTest.java | 3 +
.../ParallelIndexSupervisorTaskKillTest.java | 1 +
.../ParallelIndexSupervisorTaskResourceTest.java | 1 +
.../ParallelIndexSupervisorTaskSerdeTest.java | 1 +
.../parallel/ParallelIndexSupervisorTaskTest.java | 1 +
.../parallel/ParallelIndexTestingFactory.java | 1 +
.../parallel/ParallelIndexTuningConfigTest.java | 7 +
.../parallel/SinglePhaseParallelIndexingTest.java | 1 +
.../firehose/IngestSegmentFirehoseFactoryTest.java | 2 +-
.../IngestSegmentFirehoseFactoryTimelineTest.java | 2 +-
.../druid/indexing/overlord/TaskLifecycleTest.java | 8 +-
.../indexing/worker/WorkerTaskManagerTest.java | 2 +-
.../indexing/worker/WorkerTaskMonitorTest.java | 2 +-
.../clients/OverlordResourceTestClient.java | 2 +-
.../coordinator/duty/ITAutoCompactionTest.java | 5 +
.../hadoop/AbstractAzureInputHadoopIndexTest.java | 6 +-
.../hadoop/AbstractGcsInputHadoopIndexTest.java | 6 +-
.../hadoop/AbstractS3InputHadoopIndexTest.java | 6 +-
.../hadoop/ITAzureInputToAzureHadoopIndexTest.java | 3 +-
.../hadoop/ITAzureInputToHdfsHadoopIndexTest.java | 3 +-
.../hadoop/ITGcsInputToGcsHadoopIndexTest.java | 3 +-
.../hadoop/ITGcsInputToHdfsHadoopIndexTest.java | 3 +-
.../druid/tests/hadoop/ITHadoopIndexTest.java | 142 ++++++++++++-
.../hadoop/ITS3InputToHdfsHadoopIndexTest.java | 3 +-
.../tests/hadoop/ITS3InputToS3HadoopIndexTest.java | 3 +-
.../AbstractAzureInputSourceParallelIndexTest.java | 8 +-
.../AbstractGcsInputSourceParallelIndexTest.java | 8 +-
.../AbstractHdfsInputSourceParallelIndexTest.java | 9 +-
.../tests/indexer/AbstractITBatchIndexTest.java | 94 +++++++--
.../druid/tests/indexer/AbstractIndexerTest.java | 5 +
.../AbstractLocalInputSourceParallelIndexTest.java | 17 +-
.../AbstractOssInputSourceParallelIndexTest.java | 8 +-
.../AbstractS3InputSourceParallelIndexTest.java | 8 +-
.../tests/indexer/ITAppendBatchIndexTest.java | 11 +-
.../indexer/ITAzureToAzureParallelIndexTest.java | 2 +-
.../indexer/ITAzureToHdfsParallelIndexTest.java | 2 +-
.../ITBestEffortRollupParallelIndexTest.java | 140 ++++++++++++-
.../ITCombiningFirehoseFactoryIndexTest.java | 22 +-
.../ITCombiningInputSourceParallelIndexTest.java | 7 +-
.../druid/tests/indexer/ITCompactionTaskTest.java | 7 +
.../tests/indexer/ITGcsToGcsParallelIndexTest.java | 2 +-
.../indexer/ITGcsToHdfsParallelIndexTest.java | 2 +-
.../indexer/ITHdfsToAzureParallelIndexTest.java | 2 +-
.../indexer/ITHdfsToGcsParallelIndexTest.java | 2 +-
.../indexer/ITHdfsToHdfsParallelIndexTest.java | 6 +-
.../tests/indexer/ITHdfsToS3ParallelIndexTest.java | 2 +-
.../druid/tests/indexer/ITHttpInputSourceTest.java | 4 +-
.../apache/druid/tests/indexer/ITIndexerTest.java | 141 ++++++++++++-
.../ITLocalInputSourceAllInputFormatTest.java | 15 +-
.../tests/indexer/ITOssToOssParallelIndexTest.java | 2 +-
.../tests/indexer/ITOverwriteBatchIndexTest.java | 4 +-
.../indexer/ITPerfectRollupParallelIndexTest.java | 16 +-
.../indexer/ITS3OverrideCredentialsIndexTest.java | 4 +-
.../tests/indexer/ITS3ToHdfsParallelIndexTest.java | 2 +-
.../tests/indexer/ITS3ToS3ParallelIndexTest.java | 2 +-
.../druid/tests/indexer/ITSqlInputSourceTest.java | 4 +-
.../indexer/ITSystemTableBatchIndexTaskTest.java | 20 +-
.../tests/indexer/ITTestCoordinatorPausedTest.java | 22 +-
.../druid/tests/indexer/ITTransformTest.java | 16 +-
.../hadoop/wikipedia_hadoop_index_task.json | 1 +
.../resources/indexer/wikipedia_index_task.json | 3 +-
.../indexer/wikipedia_parallel_index_task.json | 3 +-
.../CoordinatorBasedSegmentHandoffNotifier.java | 8 +-
...ordinatorBasedSegmentHandoffNotifierConfig.java | 2 +-
...rdinatorBasedSegmentHandoffNotifierFactory.java | 2 +-
.../SegmentHandoffNotifier.java | 2 +-
.../SegmentHandoffNotifierFactory.java | 2 +-
.../druid/segment/indexing/TuningConfig.java | 1 +
.../realtime/appenderator/AppenderatorPlumber.java | 2 +-
.../appenderator/AppenderatorPlumberSchool.java | 2 +-
.../appenderator/StreamAppenderatorDriver.java | 4 +-
.../plumber/NoopSegmentHandoffNotifierFactory.java | 2 +
.../segment/realtime/plumber/RealtimePlumber.java | 1 +
.../realtime/plumber/RealtimePlumberSchool.java | 1 +
...CoordinatorBasedSegmentHandoffNotifierTest.java | 2 +-
.../appenderator/AppenderatorPlumberTest.java | 4 +-
.../appenderator/StreamAppenderatorDriverTest.java | 4 +-
.../plumber/RealtimePlumberSchoolTest.java | 2 +
.../main/java/org/apache/druid/cli/CliPeon.java | 6 +-
website/.spelling | 2 +
120 files changed, 1868 insertions(+), 198 deletions(-)
create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/common/task/CompactionTuningConfigTest.java
rename server/src/main/java/org/apache/druid/segment/{realtime/plumber => handoff}/CoordinatorBasedSegmentHandoffNotifier.java (94%)
rename server/src/main/java/org/apache/druid/segment/{realtime/plumber => handoff}/CoordinatorBasedSegmentHandoffNotifierConfig.java (95%)
rename server/src/main/java/org/apache/druid/segment/{realtime/plumber => handoff}/CoordinatorBasedSegmentHandoffNotifierFactory.java (96%)
rename server/src/main/java/org/apache/druid/segment/{realtime/plumber => handoff}/SegmentHandoffNotifier.java (97%)
rename server/src/main/java/org/apache/druid/segment/{realtime/plumber => handoff}/SegmentHandoffNotifierFactory.java (94%)
rename server/src/test/java/org/apache/druid/segment/{realtime/plumber => handoff}/CoordinatorBasedSegmentHandoffNotifierTest.java (99%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org