You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ji...@apache.org on 2019/11/15 17:22:22 UTC
[incubator-druid] branch master updated (ce4ee42 -> 16117928)
This is an automated email from the ASF dual-hosted git repository.
jihoonson pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-druid.git.
from ce4ee42 Fix LIKE filter wildcards to match newlines. (#8863)
add 16117928 Add InputSource and InputFormat interfaces (#8823)
No new revisions were added by this update.
Summary of changes:
.../druid/data/input/AbstractInputSource.java | 65 +++++
.../druid/data/input/FiniteFirehoseFactory.java | 3 +
.../java/org/apache/druid/data/input/Firehose.java | 6 +-
.../apache/druid/data/input/FirehoseFactory.java | 2 +
.../input/FirehoseFactoryToInputSourceAdaptor.java | 99 +++++++
.../org/apache/druid/data/input/InputEntity.java | 120 +++++++++
.../apache/druid/data/input/InputEntityReader.java | 50 ++++
.../org/apache/druid/data/input/InputFormat.java | 57 ++++
...utRowPlusRaw.java => InputRowListPlusJson.java} | 61 +++--
.../apache/druid/data/input/InputRowSchema.java | 45 ++--
.../org/apache/druid/data/input/InputSource.java | 81 ++++++
.../apache/druid/data/input/InputSourceReader.java | 43 +++
.../data/input/IntermediateRowParsingReader.java | 101 +++++++
.../org/apache/druid/data/input/SplitHintSpec.java | 3 +
.../org/apache/druid/data/input/TextReader.java | 132 +++++++++
.../apache/druid/data/input/impl/ByteEntity.java | 54 ++--
.../apache/druid/data/input/impl/CSVParseSpec.java | 19 +-
.../druid/data/input/impl/CsvInputFormat.java | 152 +++++++++++
.../apache/druid/data/input/impl/CsvReader.java | 126 +++++++++
.../apache/druid/data/input/impl/FileEntity.java | 78 ++++++
.../data/input/impl/FileIteratingFirehose.java | 8 +-
.../impl/FirehoseToInputSourceReaderAdaptor.java | 123 +++++++++
.../apache/druid/data/input/impl/HttpEntity.java | 87 ++++++
.../druid/data/input/impl/HttpInputSource.java | 149 +++++++++++
.../input/impl/InputEntityIteratingReader.java | 96 +++++++
.../druid/data/input/impl/InputRowParser.java | 1 +
.../druid/data/input/impl/JSONParseSpec.java | 7 +
.../{JSONParseSpec.java => JsonInputFormat.java} | 69 ++---
.../apache/druid/data/input/impl/JsonReader.java | 82 ++++++
.../druid/data/input/impl/LocalInputSource.java | 144 ++++++++++
.../druid/data/input/impl/MapInputRowParser.java | 49 +++-
.../input/impl/NestedInputFormat.java} | 60 ++---
.../apache/druid/data/input/impl/ParseSpec.java | 14 +
.../data/input/impl/SplittableInputSource.java | 70 +++++
.../druid/data/input/impl/TimestampSpec.java | 7 +-
.../druid/guice/annotations/ExtensionPoint.java | 1 +
.../apache/druid/guice/annotations/PublicApi.java | 1 +
.../{ExtensionPoint.java => UnstableApi.java} | 20 +-
.../main/java/org/apache/druid/indexer/Checks.java | 2 +-
.../java/org/apache/druid/indexer/Property.java | 5 +-
.../apache/druid/java/util/common/FileUtils.java | 33 ++-
.../parsers/AbstractFlatTextFormatParser.java | 18 +-
.../util/common/parsers/CloseableIterator.java | 74 ++++++
.../org/apache/druid/timeline/DataSegment.java | 1 +
.../FirehoseFactoryToInputSourceAdaptorTest.java | 180 +++++++++++++
.../druid/data/input/impl/CsvInputFormatTest.java | 54 ++++
.../druid/data/input/impl/CsvReaderTest.java | 292 ++++++++++++++++++++
.../druid/data/input/impl/HttpInputSourceTest.java | 47 ++++
.../input/impl/InputEntityIteratingReaderTest.java | 97 +++++++
.../druid/data/input/impl/JsonInputFormatTest.java | 59 +++++
.../druid/data/input/impl/JsonReaderTest.java | 131 +++++++++
.../data/input/impl/LocalInputSourceTest.java} | 24 +-
.../druid/data/input/impl/NoopFirehoseFactory.java | 44 +--
.../druid/data/input/impl/NoopInputFormat.java} | 28 +-
.../druid/data/input/impl/NoopInputSource.java | 47 ++--
.../java/org/apache/druid/indexer/ChecksTest.java | 2 +-
.../java/util/common/parsers/CSVParserTest.java | 98 -------
docs/ingestion/tasks.md | 2 +-
extensions-core/kafka-indexing-service/pom.xml | 7 +
.../druid/indexing/kafka/KafkaIndexTask.java | 2 +-
extensions-core/kinesis-indexing-service/pom.xml | 7 +
.../druid/indexing/kinesis/KinesisIndexTask.java | 3 +-
.../parquet/simple/ParquetGroupConverter.java | 2 +-
.../druid/indexer/HadoopDruidIndexerConfig.java | 4 +-
.../apache/druid/indexer/IndexGeneratorJob.java | 22 +-
.../apache/druid/indexing/common/TaskToolbox.java | 4 +-
.../task/AppenderatorDriverRealtimeIndexTask.java | 8 +-
.../druid/indexing/common/task/IndexTask.java | 179 ++++++++++---
...oseProcessor.java => InputSourceProcessor.java} | 47 +++-
.../indexing/common/task/RealtimeIndexTask.java | 7 +-
.../task/batch/parallel/ParallelIndexIOConfig.java | 13 +-
.../batch/parallel/ParallelIndexIngestionSpec.java | 17 ++
.../batch/parallel/ParallelIndexPhaseRunner.java | 23 +-
.../parallel/ParallelIndexSupervisorTask.java | 29 +-
...tialSegmentGenerateParallelIndexTaskRunner.java | 37 ++-
.../batch/parallel/PartialSegmentGenerateTask.java | 27 +-
.../batch/parallel/PartialSegmentMergeTask.java | 2 +-
.../SinglePhaseParallelIndexTaskRunner.java | 33 ++-
.../task/batch/parallel/SinglePhaseSubTask.java | 45 +++-
.../firehose/IngestSegmentFirehoseFactory.java | 1 -
.../indexing/overlord/sampler/FirehoseSampler.java | 4 +-
.../indexing/overlord/sampler/SamplerCache.java | 8 +-
.../indexing/overlord/sampler/SamplerConfig.java | 3 +-
.../seekablestream/SeekableStreamSamplerSpec.java | 14 +-
.../apache/druid/indexing/common/TestFirehose.java | 8 +-
.../apache/druid/indexing/common/TestUtils.java | 9 +-
.../common/task/CompactionTaskRunTest.java | 12 +-
.../indexing/common/task/CompactionTaskTest.java | 12 +-
.../common/task/IndexIngestionSpecTest.java | 142 ++++++++++
.../druid/indexing/common/task/IndexTaskTest.java | 294 +++++++++++----------
.../druid/indexing/common/task/TaskSerdeTest.java | 30 ++-
.../AbstractParallelIndexSupervisorTaskTest.java | 19 +-
.../parallel/MultiPhaseParallelIndexingTest.java | 116 ++++----
.../ParallelIndexSupervisorTaskKillTest.java | 63 +++--
.../ParallelIndexSupervisorTaskResourceTest.java | 85 +++---
.../ParallelIndexSupervisorTaskSerdeTest.java | 42 +--
.../parallel/SinglePhaseParallelIndexingTest.java | 172 ++++++------
.../druid/indexing/overlord/TaskLifecycleTest.java | 145 +++++++---
.../overlord/sampler/SamplerCacheTest.java | 6 +-
.../incremental/IncrementalIndexSchema.java | 7 +-
.../druid/segment/transform/TransformSpec.java | 8 +-
.../druid/segment/transform/Transformer.java | 38 ++-
.../transform/TransformingInputRowParser.java | 5 -
.../segment/transform/TransformingReader.java | 51 ++++
.../druid/segment/indexing/BatchIOConfig.java | 14 +-
.../apache/druid/segment/indexing/DataSchema.java | 247 +++++++++++------
.../druid/segment/realtime/FireDepartment.java | 3 +-
.../realtime/firehose/HttpFirehoseFactory.java | 20 +-
.../segment/realtime/firehose/InlineFirehose.java | 8 +-
.../realtime/firehose/SqlFirehoseFactory.java | 4 +-
.../firehose/TimedShutoffFirehoseFactory.java | 4 +-
.../druid/segment/realtime/plumber/Sink.java | 4 +-
.../druid/segment/indexing/DataSchemaTest.java | 7 +-
.../segment/indexing/TestModifiedDataSchema.java | 20 +-
.../realtime/firehose/InlineFirehoseTest.java | 8 +-
.../druid/segment/realtime/plumber/SinkTest.java | 15 +-
116 files changed, 4606 insertions(+), 1083 deletions(-)
create mode 100644 core/src/main/java/org/apache/druid/data/input/AbstractInputSource.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/FirehoseFactoryToInputSourceAdaptor.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/InputEntity.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/InputEntityReader.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/InputFormat.java
rename core/src/main/java/org/apache/druid/data/input/{InputRowPlusRaw.java => InputRowListPlusJson.java} (56%)
copy indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexIOConfig.java => core/src/main/java/org/apache/druid/data/input/InputRowSchema.java (50%)
create mode 100644 core/src/main/java/org/apache/druid/data/input/InputSource.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/InputSourceReader.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/IntermediateRowParsingReader.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/TextReader.java
copy indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexIOConfig.java => core/src/main/java/org/apache/druid/data/input/impl/ByteEntity.java (51%)
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/CsvInputFormat.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/CsvReader.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/FileEntity.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/FirehoseToInputSourceReaderAdaptor.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/HttpEntity.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/HttpInputSource.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/InputEntityIteratingReader.java
copy core/src/main/java/org/apache/druid/data/input/impl/{JSONParseSpec.java => JsonInputFormat.java} (53%)
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/JsonReader.java
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/LocalInputSource.java
copy core/src/main/java/org/apache/druid/{indexer/Property.java => data/input/impl/NestedInputFormat.java} (53%)
create mode 100644 core/src/main/java/org/apache/druid/data/input/impl/SplittableInputSource.java
copy core/src/main/java/org/apache/druid/guice/annotations/{ExtensionPoint.java => UnstableApi.java} (53%)
create mode 100644 core/src/test/java/org/apache/druid/data/input/FirehoseFactoryToInputSourceAdaptorTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/CsvInputFormatTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/CsvReaderTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/HttpInputSourceTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/InputEntityIteratingReaderTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/JsonInputFormatTest.java
create mode 100644 core/src/test/java/org/apache/druid/data/input/impl/JsonReaderTest.java
copy core/src/{main/java/org/apache/druid/java/util/common/parsers/CloseableIterator.java => test/java/org/apache/druid/data/input/impl/LocalInputSourceTest.java} (53%)
copy indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexIOConfig.java => core/src/test/java/org/apache/druid/data/input/impl/NoopFirehoseFactory.java (52%)
copy core/src/{main/java/org/apache/druid/java/util/common/parsers/CloseableIterator.java => test/java/org/apache/druid/data/input/impl/NoopInputFormat.java} (60%)
copy indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexIOConfig.java => core/src/test/java/org/apache/druid/data/input/impl/NoopInputSource.java (53%)
delete mode 100644 core/src/test/java/org/apache/druid/java/util/common/parsers/CSVParserTest.java
rename indexing-service/src/main/java/org/apache/druid/indexing/common/task/{FiniteFirehoseProcessor.java => InputSourceProcessor.java} (82%)
create mode 100644 indexing-service/src/test/java/org/apache/druid/indexing/common/task/IndexIngestionSpecTest.java
create mode 100644 processing/src/main/java/org/apache/druid/segment/transform/TransformingReader.java
copy core/src/main/java/org/apache/druid/java/util/common/parsers/CloseableIterator.java => server/src/main/java/org/apache/druid/segment/indexing/BatchIOConfig.java (70%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org