You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by cw...@apache.org on 2026/03/17 20:11:19 UTC
(druid) branch master updated: remove support for hadoop ingestion (#19109)
This is an automated email from the ASF dual-hosted git repository.
cwylie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 01abc30bea7 remove support for hadoop ingestion (#19109)
01abc30bea7 is described below
commit 01abc30bea7d2820554fbcb73187491f8ea92c17
Author: Clint Wylie <cw...@apache.org>
AuthorDate: Tue Mar 17 13:11:11 2026 -0700
remove support for hadoop ingestion (#19109)
changes:
* removes support for hadoop ingestion and all associated hadoop specific extension functionality (materialized view extensions, hadoop parsers for avro, parquet, etc)
* adds hadoop task stub for serde and logging to indicate that hadoop ingestion support has been removed
* updates hadoop docs to indicate it is removed, redirect other hadoop specific doc pages
* remove legacy integration tests too since the only remaining tests were hadoop ingestion tests
---
.github/labeler.yml | 1 -
.github/scripts/run_docker-tests | 1 -
.github/scripts/run_unit-tests | 1 -
.github/workflows/reusable-standard-its.yml | 116 -
.github/workflows/standard-its.yml | 43 -
README.md | 4 -
codestyle/checkstyle-suppressions.xml | 3 -
dev/intellij-setup.md | 2 +-
distribution/asf-release-process-guide.md | 7 +-
distribution/bin/jar-notice-lister.py | 7 -
distribution/docker/DockerfileBuildTarAdvanced | 29 -
distribution/docker/deduplicate_jars.sh | 2 +-
distribution/docker/druid.sh | 2 +-
distribution/docker/peon.sh | 2 +-
distribution/pom.xml | 122 -
distribution/src/assembly/assembly.xml | 7 -
.../src/assembly/integration-test-assembly.xml | 7 -
docs/configuration/extensions.md | 3 -
docs/configuration/index.md | 11 +-
docs/design/segments.md | 2 +-
docs/design/storage.md | 8 +-
docs/development/extensions-contrib/delta-lake.md | 2 -
docs/development/extensions-contrib/iceberg.md | 2 -
.../extensions-contrib/materialized-view.md | 136 -
docs/development/extensions-contrib/thrift.md | 56 +-
docs/development/extensions-core/avro.md | 1 -
docs/development/extensions-core/google.md | 3 +-
docs/development/extensions-core/hdfs.md | 125 -
docs/development/extensions-core/orc.md | 55 +-
docs/development/extensions-core/parquet.md | 11 +-
docs/development/extensions-core/s3.md | 4 -
docs/development/overview.md | 7 -
docs/ingestion/data-formats.md | 603 +---
docs/ingestion/hadoop.md | 559 +---
docs/ingestion/index.md | 28 +-
docs/ingestion/ingestion-spec.md | 18 +-
docs/ingestion/native-batch.md | 4 +-
docs/ingestion/partitioning.md | 6 +-
docs/ingestion/rollup.md | 1 -
docs/ingestion/tasks.md | 13 +-
docs/operations/basic-cluster-tuning.md | 4 -
docs/operations/other-hadoop.md | 310 --
docs/operations/pull-deps.md | 44 +-
docs/operations/segment-optimization.md | 2 +-
docs/operations/use_sbt_to_build_fat_jar.md | 127 -
docs/querying/geo.md | 54 +-
docs/querying/multitenancy.md | 3 +-
docs/tutorials/cluster.md | 22 -
docs/tutorials/tutorial-batch-hadoop.md | 252 --
docs/tutorials/tutorial-kerberos-hadoop.md | 34 +-
embedded-tests/pom.xml | 21 +
.../embedded/auth/EmbeddedSSLAuthResource.java | 6 +-
.../tls/generate-client-certs-and-keystores.sh | 0
.../tls/generate-expired-client-cert.sh | 0
.../tls/generate-good-client-cert.sh | 0
.../tls/generate-incorrect-hostname-client-cert.sh | 0
.../generate-invalid-intermediate-client-cert.sh | 0
.../tls/generate-root-certs.sh | 0
.../tls/generate-server-certs-and-keystores.sh | 0
.../tls/generate-to-be-revoked-client-cert.sh | 0
.../tls/generate-untrusted-root-client-cert.sh | 0
.../tls/generate-valid-intermediate-client-cert.sh | 0
.../docker => embedded-tests}/tls/root.cnf | 0
.../docker => embedded-tests}/tls/root2.cnf | 0
.../tls/set-docker-host-ip.sh | 0
.../druid/auto/_common/common.runtime.properties | 5 -
.../conf/druid/auto/indexer/runtime.properties | 3 -
.../druid/auto/middleManager/runtime.properties | 3 -
.../cluster/_common/common.runtime.properties | 5 -
.../druid/cluster/data/indexer/runtime.properties | 3 -
.../cluster/data/middleManager/runtime.properties | 3 -
.../large/_common/common.runtime.properties | 5 -
.../large/middleManager/runtime.properties | 3 -
.../medium/_common/common.runtime.properties | 5 -
.../medium/middleManager/runtime.properties | 3 -
.../_common/common.runtime.properties | 5 -
.../middleManager/runtime.properties | 3 -
.../_common/common.runtime.properties | 5 -
.../middleManager/runtime.properties | 3 -
.../small/_common/common.runtime.properties | 4 -
.../small/middleManager/runtime.properties | 3 -
.../xlarge/_common/common.runtime.properties | 5 -
.../xlarge/middleManager/runtime.properties | 3 -
.../tutorial/wikipedia-index-hadoop3.json | 81 -
.../druid/storage/aliyun/OssDataSegmentPusher.java | 22 -
.../cassandra/CassandraDataSegmentPusher.java | 13 -
.../cloudfiles/CloudFilesDataSegmentPusher.java | 13 -
.../materialized-view-maintenance/pom.xml | 142 -
.../DerivativeDataSourceMetadata.java | 138 -
.../MaterializedViewMaintenanceDruidModule.java | 54 -
.../MaterializedViewSupervisor.java | 517 ---
.../MaterializedViewSupervisorReport.java | 63 -
.../MaterializedViewSupervisorSpec.java | 432 ---
.../MaterializedViewTaskConfig.java | 34 -
.../org.apache.druid.initialization.DruidModule | 16 -
.../DerivativeDataSourceMetadataTest.java | 61 -
.../MaterializedViewSupervisorSpecTest.java | 346 --
.../MaterializedViewSupervisorTest.java | 379 ---
.../materialized-view-selection/pom.xml | 149 -
.../materializedview/DataSourceOptimizer.java | 216 --
.../DataSourceOptimizerMonitor.java | 66 -
.../materializedview/DataSourceOptimizerStats.java | 93 -
.../materializedview/DerivativeDataSource.java | 94 -
.../DerivativeDataSourceManager.java | 254 --
.../materializedview/MaterializedViewConfig.java | 35 -
.../materializedview/MaterializedViewQuery.java | 225 --
.../MaterializedViewQueryQueryToolChest.java | 157 -
.../MaterializedViewQueryRunner.java | 68 -
.../MaterializedViewSelectionDruidModule.java | 62 -
.../materializedview/MaterializedViewUtils.java | 233 --
.../org.apache.druid.initialization.DruidModule | 16 -
.../DataSourceOptimizerMonitorTest.java | 68 -
.../materializedview/DatasourceOptimizerTest.java | 350 --
.../MaterializedViewQueryQueryToolChestTest.java | 249 --
.../MaterializedViewQueryTest.java | 126 -
.../MaterializedViewUtilsTest.java | 232 --
.../thrift-extensions/example/books.json | 65 -
extensions-contrib/thrift-extensions/pom.xml | 6 -
extensions-core/avro-extensions/pom.xml | 41 -
.../druid/data/input/AvroHadoopInputRowParser.java | 128 -
.../data/input/avro/AvroExtensionsModule.java | 3 -
.../druid/data/input/avro/AvroParseSpec.java | 65 -
.../apache/druid/data/input/avro/AvroParsers.java | 75 -
.../data/input/avro/AvroValueInputFormat.java | 71 -
.../data/input/avro/AvroValueRecordReader.java | 51 -
.../data/input/AvroHadoopInputRowParserTest.java | 154 -
.../data/input/AvroStreamInputFormatTest.java | 17 +
.../druid/data/input/avro/AvroOCFReaderTest.java | 3 +-
.../storage/azure/AzureDataSegmentPusher.java | 35 -
.../org/apache/druid/storage/azure/AzureUtils.java | 5 -
.../storage/azure/AzureDataSegmentPusherTest.java | 25 -
.../storage/google/GoogleDataSegmentPusher.java | 22 -
extensions-core/hdfs-storage/pom.xml | 6 -
.../druid/inputsource/hdfs/HdfsInputSource.java | 4 +-
.../druid/storage/hdfs/HdfsDataSegmentPusher.java | 25 -
.../storage/hdfs/HdfsDataSegmentPusherTest.java | 189 --
.../example/orc-file-11-format-hadoop-job.json | 85 -
.../example/orc_split_elim_hadoop_job.json | 64 -
.../example/testDate1900_hadoop_job.json | 64 -
.../example/testDate2038_hadoop_job.json | 64 -
.../orc-extensions/example/test_1_hadoop_job.json | 54 -
.../orc-extensions/example/test_2_hadoop_job.json | 70 -
extensions-core/orc-extensions/pom.xml | 6 -
.../druid/data/input/orc/OrcExtensionsModule.java | 2 -
.../data/input/orc/OrcHadoopInputRowParser.java | 88 -
.../apache/druid/data/input/orc/OrcParseSpec.java | 67 -
.../input/orc/OrcHadoopInputRowParserTest.java | 284 --
.../apache/druid/data/input/orc/OrcReaderTest.java | 6 -
.../example/compat/impala_hadoop_parquet_job.json | 69 -
.../example/compat/nested_array_struct.json | 90 -
.../example/compat/old_repeated_int.json | 76 -
.../example/compat/parquet_1217.json | 86 -
.../example/compat/parquet_thrift_compat.json | 98 -
.../example/compat/proto_struct_with_array.json | 90 -
.../example/decimals/dec_in_fix_len.json | 88 -
.../example/decimals/dec_in_i32.json | 88 -
.../example/decimals/dec_in_i64.json | 88 -
.../example/flattening/flat_1.json | 76 -
.../flattening/flat_1_autodiscover_fields.json | 80 -
.../example/flattening/flat_1_flatten.json | 106 -
.../example/flattening/flat_1_list_index.json | 97 -
.../example/flattening/nested_1.json | 73 -
.../flattening/nested_1_autodiscover_fields.json | 77 -
.../example/flattening/nested_1_flatten.json | 109 -
.../example/flattening/nested_1_list_index.json | 99 -
.../example/flattening/nullable_list_flatten.json | 99 -
.../timestamps/date_test_data_job_date.json | 65 -
.../timestamps/date_test_data_job_string.json | 66 -
.../example/timestamps/int96_timestamp.json | 64 -
.../example/timestamps/timemillis_in_i64.json | 70 -
.../parquet-extensions/example/wiki/wiki.json | 81 -
extensions-core/parquet-extensions/pom.xml | 35 -
.../input/parquet/DruidParquetAvroInputFormat.java | 32 -
.../input/parquet/DruidParquetInputFormat.java | 32 -
.../input/parquet/ParquetExtensionsModule.java | 6 -
.../avro/ParquetAvroHadoopInputRowParser.java | 147 -
.../parquet/simple/DruidParquetReadSupport.java | 89 -
.../simple/ParquetHadoopInputRowParser.java | 86 -
.../input/parquet/simple/ParquetParseSpec.java | 67 -
.../parquet/avro/DruidParquetAvroReadSupport.java | 130 -
.../data/input/parquet/BaseParquetInputTest.java | 139 -
.../data/input/parquet/CompatParquetInputTest.java | 255 --
.../input/parquet/CompatParquetReaderTest.java | 3 -
.../input/parquet/DecimalParquetInputTest.java | 191 --
.../input/parquet/DecimalParquetReaderTest.java | 3 -
.../input/parquet/FlattenSpecParquetInputTest.java | 248 --
.../parquet/FlattenSpecParquetReaderTest.java | 20 +-
.../parquet/NestedColumnParquetReaderTest.java | 6 +-
.../input/parquet/TimestampsParquetInputTest.java | 113 -
.../input/parquet/TimestampsParquetReaderTest.java | 3 -
.../data/input/parquet/WikiParquetInputTest.java | 75 -
.../data/input/parquet/WikiParquetReaderTest.java | 3 -
.../druid/storage/s3/S3DataSegmentPusher.java | 27 +-
.../storage/s3/S3DataSegmentPusherConfig.java | 13 -
.../storage/s3/S3DataSegmentPusherConfigTest.java | 4 +-
indexing-hadoop/pom.xml | 269 --
.../main/java/org/apache/druid/indexer/Bucket.java | 132 -
.../indexer/DataSegmentAndIndexZipFilePath.java | 100 -
.../indexer/DetermineHashedPartitionsJob.java | 494 ---
.../druid/indexer/DeterminePartitionsJob.java | 1073 -------
.../indexer/DeterminePartitionsJobSampler.java | 70 -
.../HadoopDruidDetermineConfigurationJob.java | 148 -
.../druid/indexer/HadoopDruidIndexerConfig.java | 629 ----
.../druid/indexer/HadoopDruidIndexerJob.java | 132 -
.../druid/indexer/HadoopDruidIndexerMapper.java | 151 -
.../org/apache/druid/indexer/HadoopIOConfig.java | 79 -
.../org/apache/druid/indexer/HadoopIndexTask.java | 1084 -------
.../druid/indexer/HadoopIndexTaskModule.java | 50 -
.../apache/druid/indexer/HadoopIngestionSpec.java | 241 --
.../apache/druid/indexer/HadoopKerberosConfig.java | 78 -
.../java/org/apache/druid/indexer/HadoopTask.java | 277 --
.../org/apache/druid/indexer/HadoopTaskConfig.java | 85 -
.../apache/druid/indexer/HadoopTuningConfig.java | 456 ---
.../druid/indexer/HadoopWorkingDirCleaner.java | 44 -
.../org/apache/druid/indexer/HadoopyShardSpec.java | 65 -
.../druid/indexer/HadoopyStringInputRowParser.java | 71 -
.../apache/druid/indexer/IndexGeneratorJob.java | 909 ------
.../apache/druid/indexer/IndexingHadoopModule.java | 50 -
.../org/apache/druid/indexer/InputRowSerde.java | 524 ---
.../java/org/apache/druid/indexer/JobHelper.java | 865 -----
.../druid/indexer/MetadataStorageUpdaterJob.java | 57 -
.../OverlordActionBasedUsedSegmentsRetriever.java | 58 -
.../org/apache/druid/indexer/SortableBytes.java | 154 -
.../main/java/org/apache/druid/indexer/Utils.java | 249 --
.../indexer/hadoop/DatasourceIngestionSpec.java | 264 --
.../indexer/hadoop/DatasourceInputFormat.java | 331 --
.../druid/indexer/hadoop/DatasourceInputSplit.java | 101 -
.../indexer/hadoop/DatasourceRecordReader.java | 332 --
.../druid/indexer/hadoop/FSSpideringIterator.java | 126 -
.../druid/indexer/hadoop/SegmentInputRow.java | 94 -
.../druid/indexer/hadoop/WindowedDataSegment.java | 89 -
.../org/apache/druid/indexer/package-info.java | 23 -
.../druid/indexer/path/DatasourcePathSpec.java | 230 --
.../indexer/path/GranularUnprocessedPathSpec.java | 124 -
.../druid/indexer/path/GranularityPathSpec.java | 172 -
.../druid/indexer/path/HadoopGlobPathSplitter.java | 107 -
.../MetadataStoreBasedUsedSegmentsRetriever.java | 56 -
.../druid/indexer/path/MultiplePathSpec.java | 78 -
.../org/apache/druid/indexer/path/PathSpec.java | 42 -
.../indexer/path/SegmentMetadataPublisher.java | 44 -
.../apache/druid/indexer/path/StaticPathSpec.java | 173 -
.../druid/indexer/path/UsedSegmentsRetriever.java | 60 -
.../updater/MetadataStorageUpdaterJobSpec.java | 106 -
.../src/main/resources/hadoop.indexer.libs.version | 1 -
.../druid/indexer/BatchDeltaIngestionTest.java | 555 ----
.../java/org/apache/druid/indexer/BucketTest.java | 99 -
.../DataSegmentAndIndexZipFilePathTest.java | 164 -
.../indexer/DetermineHashedPartitionsJobTest.java | 273 --
.../indexer/DeterminePartitionsJobSamplerTest.java | 102 -
.../druid/indexer/DeterminePartitionsJobTest.java | 406 ---
.../indexer/DetermineRangePartitionsJobTest.java | 451 ---
.../HadoopDruidDetermineConfigurationJobTest.java | 126 -
.../indexer/HadoopDruidIndexerConfigTest.java | 371 ---
.../druid/indexer/HadoopDruidIndexerJobTest.java | 75 -
.../indexer/HadoopDruidIndexerMapperTest.java | 275 --
.../apache/druid/indexer/HadoopIndexTaskTest.java | 125 -
.../druid/indexer/HadoopIngestionSpecTest.java | 361 ---
...onSpecUpdateDatasourcePathSpecSegmentsTest.java | 327 --
.../druid/indexer/HadoopKerberosConfigTest.java | 41 -
.../apache/druid/indexer/HadoopTaskSerdeTest.java | 184 --
.../org/apache/druid/indexer/HadoopTaskTest.java | 137 -
.../druid/indexer/HadoopTuningConfigTest.java | 102 -
.../indexer/HadoopyStringInputRowParserTest.java | 51 -
.../druid/indexer/HdfsClasspathSetupTest.java | 200 --
.../druid/indexer/IndexGeneratorCombinerTest.java | 303 --
.../druid/indexer/IndexGeneratorJobTest.java | 732 -----
.../apache/druid/indexer/InputRowSerdeTest.java | 503 ---
.../org/apache/druid/indexer/JobHelperTest.java | 335 --
.../indexer/MetadataStorageUpdaterJobTest.java | 68 -
.../apache/druid/indexer/SortableBytesTest.java | 113 -
.../apache/druid/indexer/UtilsCompressionTest.java | 131 -
.../java/org/apache/druid/indexer/UtilsTest.java | 134 -
.../hadoop/DatasourceIngestionSpecTest.java | 163 -
.../indexer/hadoop/DatasourceInputFormatTest.java | 502 ---
.../indexer/hadoop/DatasourceInputSplitTest.java | 78 -
.../DatasourceRecordReaderSegmentReaderTest.java | 242 --
.../indexer/hadoop/DatasourceRecordReaderTest.java | 146 -
.../indexer/hadoop/FSSpideringIteratorTest.java | 139 -
.../indexer/hadoop/WindowedDataSegmentTest.java | 79 -
.../partitions/HashedPartitionsSpecTest.java | 255 --
.../druid/indexer/path/DatasourcePathSpecTest.java | 352 --
.../path/GranularUnprocessedPathSpecTest.java | 35 -
.../indexer/path/GranularityPathSpecTest.java | 304 --
.../indexer/path/HadoopGlobPathSplitterTest.java | 134 -
.../druid/indexer/path/MultiplePathSpecTest.java | 65 -
.../druid/indexer/path/StaticPathSpecTest.java | 96 -
.../updater/MetadataStorageUpdaterJobSpecTest.java | 82 -
.../druid.test.data.with.duplicate.rows.tsv | 82 -
.../druid.test.data.with.rows.in.timezone.tsv | 16 -
indexing-hadoop/src/test/resources/log4j2.xml | 35 -
.../test/resources/test-segment/descriptor.json | 17 -
.../src/test/resources/test-segment/index.zip | Bin 1019 -> 0 bytes
.../src/test/resources/test-segment/note | 5 -
indexing-service/pom.xml | 5 -
.../apache/druid/indexing/common/TaskToolbox.java | 4 +-
.../druid/indexing/common/config/TaskConfig.java | 16 -
.../druid/indexing/common/task/AbstractTask.java | 1 -
.../indexing/common/task/HadoopIndexTaskStub.java | 125 +
.../druid/indexing/common/task/IndexTask.java | 6 +-
.../druid/indexing/common/task/Initialization.java | 62 -
.../apache/druid/indexing/common/task/Task.java | 5 +-
.../parallel/PartialDimensionDistributionTask.java | 1 -
.../indexing/overlord/http/OverlordResource.java | 2 +-
.../worker/shuffle/ShuffleDataSegmentPusher.java | 12 -
.../indexing/common/config/TaskConfigBuilder.java | 8 -
.../indexing/common/task/AbstractTaskTest.java | 7 -
.../common/task/HadoopIndexTaskStubTest.java | 142 +
.../indexing/common/task/InitializationTest.java | 95 -
.../druid/indexing/overlord/TaskLifecycleTest.java | 12 -
.../setup/JavaScriptWorkerSelectStrategyTest.java | 12 +-
.../druid/indexing/test/TestDataSegmentPusher.java | 12 -
.../testing/tools/ConfigFileConfigProvider.java | 2 -
.../druid/testing/tools/DockerConfigProvider.java | 3 -
integration-tests/.gitignore | 7 -
integration-tests/README.md | 526 ---
integration-tests/build_run_cluster.sh | 42 -
integration-tests/docker/Dockerfile | 136 -
integration-tests/docker/base-setup.sh | 73 -
integration-tests/docker/docker-compose.base.yml | 418 ---
...ocker-compose.centralized-datasource-schema.yml | 116 -
.../docker/docker-compose.cli-indexer.yml | 104 -
.../docker/docker-compose.druid-hadoop.yml | 28 -
.../docker/docker-compose.ldap-security.yml | 139 -
.../docker-compose.schema-registry-indexer.yml | 29 -
.../docker/docker-compose.schema-registry.yml | 29 -
.../docker/docker-compose.security.yml | 58 -
integration-tests/docker/docker-compose.yml | 106 -
integration-tests/docker/druid.sh | 111 -
.../docker/environment-configs/broker | 45 -
.../docker/environment-configs/common | 91 -
.../docker/environment-configs/common-ldap | 91 -
.../docker/environment-configs/coordinator | 46 -
.../docker/environment-configs/empty-config | 18 -
.../docker/environment-configs/historical | 35 -
.../historical-for-query-error-test | 33 -
.../docker/environment-configs/indexer | 38 -
.../docker/environment-configs/middlemanager | 41 -
.../docker/environment-configs/overlord | 37 -
.../environment-configs/override-examples/azure | 28 -
.../environment-configs/override-examples/gcs | 29 -
.../override-examples/hadoop/azure_to_azure | 38 -
.../override-examples/hadoop/azure_to_hdfs | 41 -
.../override-examples/hadoop/gcs_to_gcs | 30 -
.../override-examples/hadoop/gcs_to_hdfs | 30 -
.../override-examples/hadoop/s3_to_hdfs | 41 -
.../override-examples/hadoop/s3_to_s3 | 41 -
.../environment-configs/override-examples/hdfs | 25 -
.../environment-configs/override-examples/oss | 30 -
.../environment-configs/override-examples/s3 | 30 -
.../docker/environment-configs/router | 32 -
.../environment-configs/router-custom-check-tls | 33 -
.../environment-configs/router-no-client-auth-tls | 33 -
.../environment-configs/router-permissive-tls | 32 -
.../test-groups/prepopulated-data | 27 -
.../test-groups/shuffle-deep-store | 24 -
.../docker/ldap-configs/bootstrap.ldif | 174 -
.../docker/schema-registry/jaas_config.file | 5 -
.../docker/schema-registry/password-file | 1 -
.../docker/service-supervisords/druid.conf | 6 -
.../docker/service-supervisords/kafka.conf | 4 -
.../service-supervisords/metadata-storage.conf | 5 -
.../docker/service-supervisords/zookeeper.conf | 5 -
integration-tests/docker/supervisord.conf | 7 -
.../centralized-datasource-schema-sample-data.sql | 20 -
.../docker/test-data/query-sample-data.sql | 20 -
.../test-data/sql-input-source-sample-data.sql | 115 -
.../docker/test-data/upgrade-sample-data.sql | 16 -
integration-tests/docker/test-data/wikipedia.desc | Bin 727 -> 0 bytes
integration-tests/docker/wiki-simple-lookup.json | 1 -
integration-tests/k8s/role-and-binding.yaml | 39 -
integration-tests/k8s/tiny-cluster.yaml | 335 --
integration-tests/pom.xml | 680 ----
integration-tests/quickstart-it.json | 16 -
integration-tests/script/build_run_k8s_cluster.sh | 40 -
integration-tests/script/copy_hadoop_resources.sh | 60 -
.../script/copy_resources_template.sh | 91 -
.../script/docker_build_containers.sh | 58 -
integration-tests/script/docker_compose_args.sh | 52 -
integration-tests/script/docker_run_cluster.sh | 57 -
integration-tests/script/setup_druid_on_k8s.sh | 47 -
.../script/setup_druid_operator_on_k8s.sh | 37 -
integration-tests/script/setup_k8s_cluster.sh | 34 -
integration-tests/script/stop_k8s_cluster.sh | 30 -
.../testing/IntegrationTestingCuratorConfig.java | 46 -
.../clients/AbstractQueryResourceTestClient.java | 242 --
.../apache/druid/testing/clients/AdminClient.java | 36 -
.../clients/ClientInfoResourceTestClient.java | 100 -
.../clients/CoordinatorResourceTestClient.java | 443 ---
.../clients/OverlordResourceTestClient.java | 769 -----
.../testing/clients/QueryResourceTestClient.java | 78 -
.../testing/clients/SqlResourceTestClient.java | 44 -
.../druid/testing/clients/TaskResponseObject.java | 90 -
.../druid/testing/guice/DruidTestModule.java | 89 -
.../testing/guice/DruidTestModuleFactory.java | 63 -
.../org/apache/druid/testing/guice/TestClient.java | 36 -
.../testing/utils/AbstractTestQueryHelper.java | 188 --
.../testing/utils/DruidClusterAdminClient.java | 329 --
.../apache/druid/testing/utils/LoggerListener.java | 70 -
.../druid/testing/utils/MsqTestQueryHelper.java | 319 --
.../druid/testing/utils/ServerDiscoveryUtil.java | 66 -
.../druid/testing/utils/SqlTestQueryHelper.java | 95 -
.../druid/testing/utils/StreamAdminClient.java | 44 -
.../apache/druid/testing/utils/SuiteListener.java | 65 -
.../druid/testing/utils/TestQueryHelper.java | 86 -
.../java/org/testng/DruidTestRunnerFactory.java | 102 -
integration-tests/src/main/resources/log4j2.xml | 56 -
.../java/org/apache/druid/tests/TestNGGroup.java | 45 -
.../hadoop/AbstractAzureInputHadoopIndexTest.java | 85 -
.../hadoop/AbstractGcsInputHadoopIndexTest.java | 81 -
.../hadoop/AbstractS3InputHadoopIndexTest.java | 98 -
.../hadoop/ITAzureInputToAzureHadoopIndexTest.java | 51 -
.../hadoop/ITAzureInputToHdfsHadoopIndexTest.java | 51 -
.../hadoop/ITGcsInputToGcsHadoopIndexTest.java | 52 -
.../hadoop/ITGcsInputToHdfsHadoopIndexTest.java | 52 -
.../druid/tests/hadoop/ITHadoopIndexTest.java | 452 ---
.../hadoop/ITS3InputToHdfsHadoopIndexTest.java | 50 -
.../tests/hadoop/ITS3InputToS3HadoopIndexTest.java | 50 -
.../tests/indexer/AbstractITBatchIndexTest.java | 527 ---
.../druid/tests/indexer/AbstractIndexerTest.java | 231 --
.../batch_index/avro/wikipedia_index_data1.avro | Bin 1066 -> 0 bytes
.../batch_index/avro/wikipedia_index_data2.avro | Bin 1055 -> 0 bytes
.../batch_index/avro/wikipedia_index_data3.avro | Bin 1151 -> 0 bytes
.../data/batch_index/csv/wikipedia_index_data1.csv | 4 -
.../data/batch_index/csv/wikipedia_index_data2.csv | 4 -
.../data/batch_index/csv/wikipedia_index_data3.csv | 5 -
.../data/batch_index/hadoop_tsv/batch_hadoop.data | 3360 --------------------
.../batch_index/json/wikipedia_index_data1.json | 3 -
.../batch_index/json/wikipedia_index_data2.json | 3 -
.../batch_index/json/wikipedia_index_data3.json | 4 -
.../wikipedia_index_data1.parquet | Bin 10653 -> 0 bytes
.../wikipedia_index_data2.parquet | Bin 9935 -> 0 bytes
.../wikipedia_index_data3.parquet | Bin 9983 -> 0 bytes
.../data/batch_index/orc/wikipedia_index_data1.orc | Bin 2035 -> 0 bytes
.../data/batch_index/orc/wikipedia_index_data2.orc | Bin 2026 -> 0 bytes
.../data/batch_index/orc/wikipedia_index_data3.orc | Bin 2008 -> 0 bytes
.../parquet/wikipedia_index_data1.parquet | Bin 4119 -> 0 bytes
.../parquet/wikipedia_index_data2.parquet | Bin 4116 -> 0 bytes
.../parquet/wikipedia_index_data3.parquet | Bin 4160 -> 0 bytes
.../data/batch_index/tsv/wikipedia_index_data1.tsv | 4 -
.../data/batch_index/tsv/wikipedia_index_data2.tsv | 4 -
.../data/batch_index/tsv/wikipedia_index_data3.tsv | 5 -
.../data/broadcast/wikipedia_index_data.json | 10 -
.../resources/hadoop/batch_hadoop_indexer.json | 85 -
.../resources/hadoop/batch_hadoop_queries.json | 309 --
...dia_hadoop_azure_input_index_task_template.json | 108 -
.../wikipedia_hadoop_gcs_input_index_task.json | 114 -
.../hadoop/wikipedia_hadoop_index_task.json | 103 -
.../wikipedia_hadoop_paquet_parser_index_data.json | 131 -
.../wikipedia_hadoop_paquet_parser_query_data.json | 157 -
.../hadoop/wikipedia_hadoop_reindex_task.json | 78 -
...ipedia_hadoop_s3_input_index_task_template.json | 115 -
.../resources/indexer/json_path_index_queries.json | 49 -
.../resources/indexer/json_path_index_task.json | 80 -
.../resources/indexer/segment_metadata_query.json | 16 -
.../indexer/sys_segment_batch_index_queries.json | 35 -
.../indexer/wikipedia_cloud_index_task.json | 87 -
.../indexer/wikipedia_cloud_simple_index_task.json | 81 -
.../indexer/wikipedia_combining_index_data.json | 3 -
...combining_input_source_index_parallel_task.json | 98 -
...pedia_combining_input_source_index_queries.json | 141 -
.../wikipedia_compaction_task_parallel.json | 22 -
...edia_compaction_task_with_granularity_spec.json | 16 -
...a_compaction_task_with_segment_granularity.json | 16 -
.../wikipedia_http_inputsource_queries.json | 47 -
.../indexer/wikipedia_http_inputsource_task.json | 74 -
.../wikipedia_index_no_rollup_preserve_metric.json | 76 -
.../resources/indexer/wikipedia_index_queries.json | 150 -
.../wikipedia_index_queries_only_data3.json | 198 --
.../wikipedia_index_queries_with_transform.json | 62 -
.../wikipedia_index_rollup_preserve_metric.json | 95 -
.../indexer/wikipedia_index_rollup_queries.json | 56 -
.../indexer/wikipedia_index_sketch_queries.json | 49 -
.../resources/indexer/wikipedia_index_task.json | 90 -
.../wikipedia_index_task_with_dimension_spec.json | 89 -
...wikipedia_index_task_with_granularity_spec.json | 69 -
...edia_index_task_with_inputsource_transform.json | 103 -
...kipedia_index_with_merge_column_limit_task.json | 84 -
.../indexer/wikipedia_merge_index_queries.json | 78 -
.../indexer/wikipedia_merge_index_task.json | 98 -
...edia_merge_reindex_druid_input_source_task.json | 93 -
.../indexer/wikipedia_merge_reindex_task.json | 93 -
...dia_parallel_druid_input_source_index_task.json | 64 -
.../indexer/wikipedia_parallel_index_queries.json | 71 -
.../indexer/wikipedia_parallel_index_task.json | 75 -
...a_parallel_index_using_sqlinputsource_task.json | 88 -
...kipedia_parallel_ingest_segment_index_task.json | 67 -
.../wikipedia_parallel_reindex_queries.json | 18 -
.../indexer/wikipedia_parallel_reindex_task.json | 73 -
...ipedia_realtime_appenderator_index_queries.json | 87 -
.../indexer/wikipedia_realtime_index_queries.json | 147 -
.../wikipedia_reindex_druid_input_source_task.json | 51 -
...ex_druid_input_source_task_with_transforms.json | 106 -
.../indexer/wikipedia_reindex_queries.json | 66 -
.../wikipedia_reindex_queries_with_transforms.json | 80 -
.../resources/indexer/wikipedia_reindex_task.json | 51 -
.../wikipedia_with_timestamp_index_task.json | 89 -
.../resources/indexer/wikiticker_index_task.json | 63 -
.../test/resources/queries/wiki-lookup-config.json | 19 -
.../queries/wikipedia_editstream_queries.json | 2045 ------------
.../results/auth_test_sys_schema_segments.json | 24 -
.../auth_test_sys_schema_server_segments.json | 6 -
.../results/auth_test_sys_schema_servers.json | 34 -
.../results/auth_test_sys_schema_tasks.json | 18 -
.../data/avro/input_format/input_format.json | 81 -
.../stream/data/avro/parser/input_row_parser.json | 39 -
.../stream/data/avro/serializer/serializer.json | 3 -
.../input_format/input_format.json | 15 -
.../parser/input_row_parser.json | 21 -
.../serializer/serializer.json | 3 -
.../stream/data/csv/input_format/input_format.json | 4 -
.../stream/data/csv/parser/input_row_parser.json | 16 -
.../stream/data/csv/serializer/serializer.json | 3 -
.../data/json/input_format/input_format.json | 3 -
.../stream/data/json/parser/input_row_parser.json | 15 -
.../stream/data/json/serializer/serializer.json | 3 -
.../data/protobuf/input_format/input_format.json | 12 -
.../data/protobuf/parser/input_row_parser.json | 18 -
.../data/protobuf/serializer/serializer.json | 3 -
.../input_format/input_format.json | 15 -
.../parser/input_row_parser.json | 21 -
.../serializer/serializer.json | 3 -
.../stream/data/supervisor_spec_template.json | 60 -
.../supervisor_with_autoscaler_spec_template.json | 74 -
..._with_idle_behaviour_enabled_spec_template.json | 63 -
.../stream/data/supervisor_with_long_duration.json | 58 -
.../stream/data/tsv/input_format/input_format.json | 4 -
.../stream/data/tsv/parser/input_row_parser.json | 16 -
.../stream/data/tsv/serializer/serializer.json | 3 -
.../stream/queries/stream_index_queries.json | 92 -
integration-tests/src/test/resources/testng.xml | 40 -
integration-tests/stop_cluster.sh | 50 -
.../msq/test/MSQTestDelegateDataSegmentPusher.java | 12 -
pom.xml | 30 -
.../data/input/impl/DelimitedInputFormat.java | 2 -
.../org/apache/druid/guice/ExtensionsConfig.java | 28 -
.../partitions/DimensionRangePartitionsSpec.java | 2 +-
.../indexer/partitions/DynamicPartitionsSpec.java | 2 +-
.../indexer/partitions/HashedPartitionsSpec.java | 4 +-
.../druid/indexer/partitions/PartitionsSpec.java | 2 +-
.../org/apache/druid/segment/IndexMergerV9.java | 7 +-
.../druid/segment/loading/DataSegmentPusher.java | 22 -
.../segment/loading/NoopDataSegmentPusher.java | 12 -
.../apache/druid/guice/ExtensionsConfigTest.java | 8 -
.../druid/initialization/CoreInjectorBuilder.java | 13 +-
.../initialization/ServerInjectorBuilder.java | 35 +-
.../segment/loading/LocalDataSegmentPusher.java | 13 -
.../loading/LocalDataSegmentPusherTest.java | 24 -
.../appenderator/BatchAppenderatorTester.java | 13 -
.../appenderator/StreamAppenderatorTester.java | 12 -
services/pom.xml | 11 -
.../org/apache/druid/cli/CliHadoopIndexer.java | 126 -
.../main/java/org/apache/druid/cli/CliIndexer.java | 2 -
.../apache/druid/cli/CliInternalHadoopIndexer.java | 174 -
.../org/apache/druid/cli/CliMiddleManager.java | 2 -
.../java/org/apache/druid/cli/CliOverlord.java | 2 -
.../main/java/org/apache/druid/cli/CliPeon.java | 2 -
.../src/main/java/org/apache/druid/cli/Main.java | 7 +-
.../org/apache/druid/cli/PullDependencies.java | 50 +-
.../druid/cli/validate/DruidJsonValidator.java | 5 -
.../org/apache/druid/cli/PullDependenciesTest.java | 138 +-
.../overlord-dynamic-config-completions.ts | 1 -
.../load-data-view/ingestion-spec-completions.ts | 1 -
website/.spelling | 9 +-
website/redirects.js | 16 +
website/sidebars.json | 13 +-
565 files changed, 473 insertions(+), 56160 deletions(-)
diff --git a/.github/labeler.yml b/.github/labeler.yml
index 341cff1da3f..5f5530e8b8e 100644
--- a/.github/labeler.yml
+++ b/.github/labeler.yml
@@ -23,7 +23,6 @@
'Area - Batch Ingestion':
- changed-files:
- any-glob-to-any-file:
- - 'indexing-hadoop/**'
- 'multi-stage-query/**'
'Area - Dependencies':
diff --git a/.github/scripts/run_docker-tests b/.github/scripts/run_docker-tests
index 0a7d2874718..17bf110c031 100755
--- a/.github/scripts/run_docker-tests
+++ b/.github/scripts/run_docker-tests
@@ -35,5 +35,4 @@ fi
# No snapshot updates
OPTS+=" -nsu"
-OPTS+=" -pl !integration-tests"
mvn -B $OPTS verify -Pdocker-tests,skip-static-checks -DskipUTs -D$DRUID_IMAGE_SYS_PROPERTY=$DRUID_IMAGE_NAME "-DjfrProfilerArgLine=$JFR_PROFILER_ARG_LINE" "$@"
diff --git a/.github/scripts/run_unit-tests b/.github/scripts/run_unit-tests
index fcd572689ba..df685a54c52 100755
--- a/.github/scripts/run_unit-tests
+++ b/.github/scripts/run_unit-tests
@@ -18,7 +18,6 @@
set -e
set -x
-OPTS+="-pl !integration-tests"
OPTS+=" -Dsurefire.failIfNoSpecifiedTests=false -P skip-static-checks -Dweb.console.skip=true"
OPTS+=" -Djacoco.destFile=target/jacoco-${HASH}.exec"
diff --git a/.github/workflows/reusable-standard-its.yml b/.github/workflows/reusable-standard-its.yml
deleted file mode 100644
index 0e5091eea37..00000000000
--- a/.github/workflows/reusable-standard-its.yml
+++ /dev/null
@@ -1,116 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name: "Standard ITs shared workflow"
-on:
- workflow_call:
- inputs:
- runtime_jdk:
- description: 'Which JDK version to use at runtime'
- required: true
- type: string
- use_indexer:
- description: 'Which indexer to use'
- required: true
- type: string
- override_config_path:
- description: 'Existing Druid configuration (using Docker) can be overridden or new config vars can be set e.g. when using cloud storage'
- required: false
- type: string
- group:
- required: true
- type: string
- description: 'Name of group of tests running (to display)'
- testing_groups:
- required: true
- type: string
- build_jdk:
- description: 'Which jdk version was used to build'
- required: true
- type: string
- mysql_driver:
- description: 'MySQL driver to use'
- required: false
- type: string
- default: com.mysql.jdbc.Driver
-
-env:
- MVN: mvn --no-snapshot-updates
- MAVEN_SKIP: -P skip-static-checks -Dweb.console.skip=true -Dmaven.javadoc.skip=true
- MAVEN_SKIP_TESTS: -P skip-tests
- DOCKER_IP: 127.0.0.1 # for integration tests
- MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
- SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
-
-jobs:
- test: # Github job that runs a given standard/old IT for a given testing group
- name: ${{ inputs.group }} integration test (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
- runs-on: ubuntu-22.04
- steps:
- - name: Checkout branch
- uses: actions/checkout@v6
-
- - name: setup java
- uses: actions/setup-java@v5
- with:
- java-version: ${{ inputs.runtime_jdk }}
- distribution: 'zulu'
-
- - name: Maven build
- run: |
- ./it.sh ci
-
- - name: Run IT
- id: run-it
- timeout-minutes: 90
- env:
- MYSQL_DRIVER_CLASSNAME: ${{ inputs.mysql_driver }}
- run: |
- # Debug echo
- echo "Mysql driver: ${MYSQL_DRIVER_CLASSNAME}"
- echo "${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}"
- ${MVN} verify -pl integration-tests -P integration-tests ${{ inputs.testing_groups }} -Djvm.runtime=${{ inputs.runtime_jdk }} -Dit.indexer=${{ inputs.use_indexer }} ${MAVEN_SKIP} -Doverride.config.path=${{ inputs.override_config_path }}
-
- - name: Collect docker logs on failure
- if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
- run: |
- mkdir docker-logs
- for c in $(docker ps -a --format="{{.Names}}")
- do
- docker logs $c > ./docker-logs/$c.log
- done
-
- - name: Tar docker logs
- if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
- run: tar cvzf ./docker-logs.tgz ./docker-logs
-
- - name: Upload docker logs to GitHub
- if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
- uses: actions/upload-artifact@v7
- with:
- name: IT-${{ inputs.group }} docker logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
- path: docker-logs.tgz
-
- - name: Collect service logs on failure
- if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
- run: |
- tar cvzf ./service-logs.tgz ~/shared/logs ~/shared/tasklogs
-
- - name: Upload Druid service logs to GitHub
- if: ${{ failure() && steps.run-it.conclusion == 'failure' }}
- uses: actions/upload-artifact@v7
- with:
- name: IT-${{ inputs.group }} service logs (Compile=jdk${{ inputs.build_jdk }}, Run=jdk${{ inputs.runtime_jdk }}, Indexer=${{ inputs.use_indexer }}, Mysql=${{ inputs.mysql_driver }})
- path: service-logs.tgz
diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml
deleted file mode 100644
index 1a26febe3d9..00000000000
--- a/.github/workflows/standard-its.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Github workflow that runs standard/old ITs
-
-name: "Standard ITs workflow"
-on:
- workflow_call:
- workflow_dispatch:
-
-jobs:
- changes:
- runs-on: ubuntu-latest
- # Required permissions
- permissions:
- pull-requests: read
- # Set job outputs to values from filter step
- outputs:
- # run everything if not a PR
- core: ${{ steps.filter.outputs.core || github.event_name != 'pull_request'}}
- common-extensions: ${{ steps.filter.outputs.common-extensions }}
- steps:
- - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36
- if: github.event_name == 'pull_request'
- id: filter
- with:
- filters: |
- common-extensions:
- - 'extension-core/(mysql-metadata-storage|druid-basic-security|simple-client-sslcontext|druid-testing-tools|druid-lookups-cached-global|druid-histogram|druid-datasketches|druid-parquet-extensions|druid-avro-extensions|druid-protobuf-extensions|druid-orc-extensions|druid-kafka-indexing-service|druid-s3-extensions)/**'
- core:
- - '!extension*/**'
diff --git a/README.md b/README.md
index c42cad09299..88a4a605017 100644
--- a/README.md
+++ b/README.md
@@ -29,10 +29,6 @@
| ๐ CodeQL | [](https://github.com/apache/druid/actions/workflows/codeql.yml) |
| ๐ Cron Job ITS | [](https://github.com/apache/druid/actions/workflows/cron-job-its.yml) |
| ๐ท๏ธ Labeler | [](https://github.com/apache/druid/actions/workflows/labeler.yml) |
-| โป๏ธ Reusable Revised ITS | [](https://github.com/apache/druid/actions/workflows/reusable-revised-its.yml) |
-| โป๏ธ Reusable Standard ITS | [](https://github.com/apache/druid/actions/workflows/reusable-standard-its.yml) |
-| ๐ Revised ITS | [](https://github.com/apache/druid/actions/workflows/revised-its.yml) |
-| ๐ง Standard ITS | [](https://github.com/apache/druid/actions/workflows/standard-its.yml) |
| ๐ ๏ธ Static Checks | [](https://github.com/apache/druid/actions/workflows/static-checks.yml) |
| ๐งช Unit and Integration Tests Unified | [](https://github.com/apache/druid/actions/workflows/unit-and-integration-tests-unified.yml) |
diff --git a/codestyle/checkstyle-suppressions.xml b/codestyle/checkstyle-suppressions.xml
index e4cef71d0c1..141caca479e 100644
--- a/codestyle/checkstyle-suppressions.xml
+++ b/codestyle/checkstyle-suppressions.xml
@@ -26,9 +26,6 @@
<suppressions>
<!-- See https://checkstyle.org/filters/suppressionfilter.html for examples -->
- <!-- Code copied from TestNG to apply a bugfix -->
- <suppress checks="AvoidStaticImport" files="[\\/]org[\\/]testng[\\/]" />
-
<suppress checks=".*" files="[\\/]target[\\/]generated-sources[\\/]" />
<suppress checks=".*" files="[\\/]target[\\/]generated-test-sources[\\/]" />
diff --git a/dev/intellij-setup.md b/dev/intellij-setup.md
index 108ec25c430..966c6c81920 100644
--- a/dev/intellij-setup.md
+++ b/dev/intellij-setup.md
@@ -93,7 +93,7 @@ You can configure application definitions in XML for import into IntelliJ. Below
<configuration default="false" name="Historical" type="Application" factoryName="Application">
<extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" />
<option name="MAIN_CLASS_NAME" value="org.apache.druid.cli.Main" />
- <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service= [...]
+ <option name="VM_PARAMETERS" value="-server -Duser.timezone=UTC -Dfile.encoding=UTF-8 -Xmx2G -XX:MaxJavaStackTraceDepth=9999 -XX:+UseG1GC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintAdaptiveSizePolicy -XX:+PrintReferenceGC -verbose:gc -XX:+PrintFlagsFinal -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager -Dorg.jboss.logging.provider=slf4j -Dlog4j.configurationFile=$PROJECT_DIR$/core/src/main/resources/log4j2.debug.xml -Ddruid.host=localhost -Ddruid.service= [...]
<option name="PROGRAM_PARAMETERS" value="server historical" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" />
<option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" />
diff --git a/distribution/asf-release-process-guide.md b/distribution/asf-release-process-guide.md
index 7799f9a1de5..d5cdc5c164a 100644
--- a/distribution/asf-release-process-guide.md
+++ b/distribution/asf-release-process-guide.md
@@ -130,7 +130,7 @@ For docs, please make sure to add any relevant redirects in `website/redirects.j
The only additions to the release branch after branching should be bug fixes, which should be back-ported from the master branch, via a second PR or a cherry-pick, not with a direct PR to the release branch.
Release manager must also ensure that CI is passing successfully on the release branch. Since CI on branch can contain additional tests such as ITs for different JVM flavours. (Note that CI is sometimes flaky for older branches).
-To check the CI status on a release branch, you can go to the commits page e.g. https://github.com/apache/druid/commits/24.0.0. On this page, latest commmit should show
+To check the CI status on a release branch, you can go to the commits page e.g. https://github.com/apache/druid/commits/24.0.0. On this page, latest commit should show
a green ✔ in the commit description. If the commit has a failed build, please click on red ✕ icon in the commit description to go to travis build job and investigate.
You can restart a failed build via travis if it is flaky.
@@ -225,9 +225,8 @@ At the core of these tools is a dependency registry file, `licenses.yaml`, that
| --- | --- |
| SOURCE/JAVA-CORE | Source level inclusions in the core Druid codebase |
| SOURCE/WEB-CONSOLE | Source level inclusions in the various web consoles |
-| BINARY/JAVA-CORE | Bundled binary-only dependencies for core Druid components (i.e., not an extension, not a hadoop-dependency, not web console stuff) |
+| BINARY/JAVA-CORE | Bundled binary-only dependencies for core Druid components (i.e., not an extension, not web console stuff) |
| BINARY/WEB-CONSOLE | Bundled binary-only dependencies for the various web consoles |
-| BINARY/HADOOP-CLIENT | Bundled binary-only dependencies contained in the `hadoop-client` directory |
| BINARY/EXTENSIONS/{extension-name} | Bundled binary-only dependencies for Druid extensions |
@@ -249,7 +248,7 @@ These additional tools were largely used to bootstrap the initial `LICENSE`, `LI
| tool | description |
| --- | --- |
-| [generate-license-dependency-reports](bin/generate-license-dependency-reports.py) | Point this to the Druid source root, and give it the location of a temp scratch directory, and it will output Maven dependency reports for Druid. (I believe I had to generate Maven dep report separately for hadoop-client) |
+| [generate-license-dependency-reports](bin/generate-license-dependency-reports.py) | Point this to the Druid source root, and give it the location of a temp scratch directory, and it will output Maven dependency reports for Druid. |
| [check-licenses](bin/check-licenses.py) | Checks `licenses.yaml` against the output of `generate-license-dependency-reports.py`, used by travis and `-Papache-release` when building distribution, to verify that all dependencies are present and match the versions in `licenses.yaml`. |
| [jar-notice-lister](bin/jar-notice-lister.py) | Point this to an extracted Druid binary distribution, and give it a temp scratch directory, and it will output NOTICE information for all the Druid JAR files. |
diff --git a/distribution/bin/jar-notice-lister.py b/distribution/bin/jar-notice-lister.py
index 7a8d38c265b..677da92bb31 100755
--- a/distribution/bin/jar-notice-lister.py
+++ b/distribution/bin/jar-notice-lister.py
@@ -36,13 +36,6 @@ def main():
command = "cp -r {}/* {}".format(lib_path, tmp_lib_path)
subprocess.check_output(command, shell=True).decode('UTF-8')
- # copy hadoop deps to the staging dir
- hdeps_path = druid_path + "/hadoop-dependencies"
- tmp_hdeps_path = tmp_path + "/2-hdeps"
- os.mkdir(tmp_hdeps_path)
- command = "cp -r {}/* {}".format(hdeps_path, tmp_hdeps_path)
- subprocess.check_output(command, shell=True).decode('UTF-8')
-
# copy all extension folders to the staging dir
ext_path = druid_path + "/extensions"
diff --git a/distribution/docker/DockerfileBuildTarAdvanced b/distribution/docker/DockerfileBuildTarAdvanced
deleted file mode 100644
index bd55d5e3277..00000000000
--- a/distribution/docker/DockerfileBuildTarAdvanced
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-# Using For IT on K8s
-ARG BASE_IMAGE=apache/druid
-FROM $BASE_IMAGE
-
-USER 0
-RUN mkdir -p /shared/wikiticker-it
-COPY examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz /shared/wikiticker-it/wikiticker-2015-09-12-sampled.json.gz
-COPY integration-tests/docker/wiki-simple-lookup.json /shared/wikiticker-it/wiki-simple-lookup.json
-
-USER druid
diff --git a/distribution/docker/deduplicate_jars.sh b/distribution/docker/deduplicate_jars.sh
index 336f955a54b..5b95c7d1cc3 100755
--- a/distribution/docker/deduplicate_jars.sh
+++ b/distribution/docker/deduplicate_jars.sh
@@ -48,4 +48,4 @@ while IFS= read -r jar; do
echo "Replaced duplicate $jar with symlink to ${canonical[$key]}"
fi
# Read in an order that retain core libs as original jars
-done < <(find $JAR_DIR -wholename '*/lib/*.jar' | sort ; find $JAR_DIR -wholename '*/extensions/*.jar' | sort ; find $JAR_DIR -wholename '*/hadoop-dependencies/*.jar' | sort)
+done < <(find $JAR_DIR -wholename '*/lib/*.jar' | sort ; find $JAR_DIR -wholename '*/extensions/*.jar' | sort)
diff --git a/distribution/docker/druid.sh b/distribution/docker/druid.sh
index e43ad0d9e41..bf1a5c88373 100755
--- a/distribution/docker/druid.sh
+++ b/distribution/docker/druid.sh
@@ -212,7 +212,7 @@ then
echo "$DRUID_SERVICE_LOG4J" >$SERVICE_CONF_DIR/log4j2.xml
fi
-DRUID_DIRS_TO_CREATE=${DRUID_DIRS_TO_CREATE-'var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache'}
+DRUID_DIRS_TO_CREATE=${DRUID_DIRS_TO_CREATE-'var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/segment-cache'}
if [ -n "${DRUID_DIRS_TO_CREATE}" ]
then
mkdir -p ${DRUID_DIRS_TO_CREATE}
diff --git a/distribution/docker/peon.sh b/distribution/docker/peon.sh
index 708be314843..3b0c172ed5c 100755
--- a/distribution/docker/peon.sh
+++ b/distribution/docker/peon.sh
@@ -155,7 +155,7 @@ then
echo "$DRUID_LOG4J" >$COMMON_CONF_DIR/log4j2.xml
fi
-DRUID_DIRS_TO_CREATE=${DRUID_DIRS_TO_CREATE-'var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/hadoop-tmp var/druid/segment-cache'}
+DRUID_DIRS_TO_CREATE=${DRUID_DIRS_TO_CREATE-'var/tmp var/druid/segments var/druid/indexing-logs var/druid/task var/druid/segment-cache'}
if [ -n "${DRUID_DIRS_TO_CREATE}" ]
then
mkdir -p ${DRUID_DIRS_TO_CREATE}
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 10277b34acd..4d2125cf514 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -184,9 +184,6 @@
<argument>-Ddruid.extensions.loadList=[]</argument>
<argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
</argument>
- <argument>
- -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
- </argument>
<argument>org.apache.druid.cli.Main</argument>
<argument>tools</argument>
<argument>pull-deps</argument>
@@ -195,10 +192,6 @@
<argument>${project.parent.version}</argument>
<argument>-l</argument>
<argument>${settings.localRepository}</argument>
- <argument>-h</argument>
- <argument>org.apache.hadoop:hadoop-client-api:${hadoop.compile.version}</argument>
- <argument>-h</argument>
- <argument>org.apache.hadoop:hadoop-client-runtime:${hadoop.compile.version}</argument>
<argument>-c</argument>
<argument>org.apache.druid.extensions:druid-avro-extensions</argument>
<argument>-c</argument>
@@ -385,9 +378,6 @@
<argument>-Ddruid.extensions.loadList=[]</argument>
<argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
</argument>
- <argument>
- -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
- </argument>
<argument>org.apache.druid.cli.Main</argument>
<argument>tools</argument>
<argument>pull-deps</argument>
@@ -395,7 +385,6 @@
<argument>${project.parent.version}</argument>
<argument>-l</argument>
<argument>${settings.localRepository}</argument>
- <argument>--no-default-hadoop</argument>
<argument>-c</argument>
<argument>org.apache.druid.extensions.contrib:ambari-metrics-emitter</argument>
<argument>-c</argument>
@@ -415,10 +404,6 @@
<argument>-c</argument>
<argument>org.apache.druid.extensions.contrib:kafka-emitter</argument>
<argument>-c</argument>
- <argument>org.apache.druid.extensions.contrib:materialized-view-maintenance</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions.contrib:materialized-view-selection</argument>
- <argument>-c</argument>
<argument>org.apache.druid.extensions.contrib:druid-opentsdb-emitter</argument>
<argument>-c</argument>
<argument>org.apache.druid.extensions.contrib:druid-redis-cache</argument>
@@ -470,112 +455,5 @@
</plugins>
</build>
</profile>
- <profile>
- <id>integration-test</id>
- <activation>
- <activeByDefault>false</activeByDefault>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>pull-deps</id>
- <phase>package</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- <configuration>
- <executable>${project.parent.basedir}/examples/bin/run-java</executable>
- <arguments>
- <argument>-classpath</argument>
- <classpath />
- <argument>-Ddruid.extensions.loadList=[]</argument>
- <argument>-Ddruid.extensions.directory=${project.build.directory}/extensions
- </argument>
- <argument>
- -Ddruid.extensions.hadoopDependenciesDir=${project.build.directory}/hadoop-dependencies
- </argument>
- <argument>org.apache.druid.cli.Main</argument>
- <argument>tools</argument>
- <argument>pull-deps</argument>
- <argument>--clean</argument>
- <argument>--defaultVersion</argument>
- <argument>${project.parent.version}</argument>
- <argument>-l</argument>
- <argument>${settings.localRepository}</argument>
- <argument>-h</argument>
- <argument>org.apache.hadoop:hadoop-client:${hadoop.compile.version}</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-avro-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-azure-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-datasketches</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-hdfs-storage</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-histogram</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-kafka-indexing-service</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-kinesis-indexing-service</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-lookups-cached-global</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-protobuf-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:mysql-metadata-storage</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-orc-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-parquet-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:postgresql-metadata-storage</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-s3-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-ec2-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-google-extensions</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:simple-client-sslcontext</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-basic-security</argument>
- <argument>-c</argument>
- <argument>org.apache.druid:druid-integration-tests</argument>
- <argument>-c</argument>
- <argument>org.apache.druid.extensions:druid-testing-tools</argument>
- <argument>${druid.distribution.pulldeps.opts}</argument>
- </arguments>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <executions>
- <execution>
- <id>distro-assembly</id>
- <phase>package</phase>
- <goals>
- <goal>single</goal>
- </goals>
- <configuration>
- <finalName>apache-druid-${project.version}-integration-test</finalName>
- <tarLongFileMode>posix</tarLongFileMode>
- <descriptors>
- <descriptor>src/assembly/integration-test-assembly.xml</descriptor>
- </descriptors>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
</project>
diff --git a/distribution/src/assembly/assembly.xml b/distribution/src/assembly/assembly.xml
index ff8e0d2fdd5..2b510ea85dd 100644
--- a/distribution/src/assembly/assembly.xml
+++ b/distribution/src/assembly/assembly.xml
@@ -34,13 +34,6 @@
<outputDirectory>extensions</outputDirectory>
</fileSet>
- <fileSet>
- <directory>${project.build.directory}/hadoop-dependencies</directory>
- <includes>
- <include>*/*/*</include>
- </includes>
- <outputDirectory>hadoop-dependencies</outputDirectory>
- </fileSet>
<fileSet>
<directory>../examples/conf</directory>
<includes>
diff --git a/distribution/src/assembly/integration-test-assembly.xml b/distribution/src/assembly/integration-test-assembly.xml
index 7b2e91a1886..67664d94a04 100644
--- a/distribution/src/assembly/integration-test-assembly.xml
+++ b/distribution/src/assembly/integration-test-assembly.xml
@@ -35,13 +35,6 @@
<outputDirectory>extensions</outputDirectory>
</fileSet>
- <fileSet>
- <directory>${project.build.directory}/hadoop-dependencies</directory>
- <includes>
- <include>*/*/*</include>
- </includes>
- <outputDirectory>hadoop-dependencies</outputDirectory>
- </fileSet>
<fileSet>
<directory>../examples/conf</directory>
<includes>
diff --git a/docs/configuration/extensions.md b/docs/configuration/extensions.md
index ae8d5987d24..6c802739fc4 100644
--- a/docs/configuration/extensions.md
+++ b/docs/configuration/extensions.md
@@ -96,7 +96,6 @@ All of these community extensions can be downloaded using [pull-deps](../operati
|kafka-emitter|Kafka metrics emitter|[link](../development/extensions-contrib/kafka-emitter.md)|
|druid-thrift-extensions|Support thrift ingestion |[link](../development/extensions-contrib/thrift.md)|
|druid-opentsdb-emitter|OpenTSDB metrics emitter |[link](../development/extensions-contrib/opentsdb-emitter.md)|
-|materialized-view-selection, materialized-view-maintenance|Materialized View|[link](../development/extensions-contrib/materialized-view.md)|
|druid-moving-average-query|Support for [Moving Average](https://en.wikipedia.org/wiki/Moving_average) and other Aggregate [Window Functions](https://en.wikibooks.org/wiki/Structured_Query_Language/Window_functions) in Druid queries.|[link](../development/extensions-contrib/moving-average-query.md)|
|druid-influxdb-emitter|InfluxDB metrics emitter|[link](../development/extensions-contrib/influxdb-emitter.md)|
|druid-momentsketch|Support for approximate quantile queries using the [momentsketch](https://github.com/stanford-futuredata/momentsketch) library|[link](../development/extensions-contrib/momentsketch-quantiles.md)|
@@ -153,9 +152,7 @@ for the (hypothetical) extension *com.example:druid-example-extension:1.0.0*, ru
java \
-cp "lib/*" \
-Ddruid.extensions.directory="extensions" \
- -Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" \
org.apache.druid.cli.Main tools pull-deps \
- --no-default-hadoop \
-c "com.example:druid-example-extension:1.0.0"
```
diff --git a/docs/configuration/index.md b/docs/configuration/index.md
index ece6293e5c5..a8b50b5bf23 100644
--- a/docs/configuration/index.md
+++ b/docs/configuration/index.md
@@ -115,12 +115,9 @@ Many of Druid's external dependencies can be plugged in as modules. Extensions c
|Property|Description|Default|
|--------|-----------|-------|
|`druid.extensions.directory`|The root extension directory where user can put extensions related files. Druid will load extensions stored under this directory.|`extensions` (This is a relative path to Druid's working directory)|
-|`druid.extensions.hadoopDependenciesDir`|The root Hadoop dependencies directory where user can put Hadoop related dependencies files. Druid will load the dependencies based on the Hadoop coordinate specified in the Hadoop index task.|`hadoop-dependencies` (This is a relative path to Druid's working directory|
|`druid.extensions.loadList`|A JSON array of extensions to load from extension directories by Druid. If it is not specified, its value will be `null` and Druid will load all the extensions under `druid.extensions.directory`. If its value is empty list `[]`, then no extensions will be loaded at all. It is also allowed to specify absolute path of other custom extensions not stored in the common extensions directory.|null|
|`druid.extensions.searchCurrentClassloader`|This is a boolean flag that determines if Druid will search the main classloader for extensions. It defaults to true but can be turned off if you have reason to not automatically add all modules on the classpath.|true|
|`druid.extensions.useExtensionClassloaderFirst`|This is a boolean flag that determines if Druid extensions should prefer loading classes from their own jars rather than jars bundled with Druid. If false, extensions must be compatible with classes provided by any jars bundled with Druid. If true, extensions may depend on conflicting versions.|false|
-|`druid.extensions.hadoopContainerDruidClasspath`|Hadoop Indexing launches Hadoop jobs and this configuration provides way to explicitly set the user classpath for the Hadoop job. By default, this is computed automatically by Druid based on the Druid service classpath and set of extensions. However, sometimes you might want to be explicit to resolve dependency conflicts between Druid and Hadoop.|null|
-|`druid.extensions.addExtensionsToHadoopContainer`|Only applicable if `druid.extensions.hadoopContainerDruidClasspath` is provided. If set to true, then extensions specified in the loadList are added to Hadoop container classpath. Note that when `druid.extensions.hadoopContainerDruidClasspath` is not provided then extensions are always added to Hadoop container classpath.|false|
### Modules
@@ -432,7 +429,6 @@ The below table shows some important configurations for S3. See [S3 Deep Storage
|`druid.storage.sse.type`|Server-side encryption type. Should be one of `s3`, `kms`, and `custom`. See the below [Server-side encryption section](../development/extensions-core/s3.md#server-side-encryption) for more details.|None|
|`druid.storage.sse.kms.keyId`|AWS KMS key ID. This is used only when `druid.storage.sse.type` is `kms` and can be empty to use the default key ID.|None|
|`druid.storage.sse.custom.base64EncodedKey`|Base64-encoded key. Should be specified if `druid.storage.sse.type` is `custom`.|None|
-|`druid.storage.useS3aSchema`|If true, use the "s3a" filesystem when using Hadoop-based ingestion. If false, the "s3n" filesystem will be used. Only affects Hadoop-based ingestion.|false|
#### HDFS deep storage
@@ -1259,7 +1255,7 @@ The following example shows a function that sends `batch_index_task` to workers
```json
{
"type":"javascript",
- "function":"function (config, zkWorkers, task) {\nvar batch_workers = new java.util.ArrayList();\nbatch_workers.add(\"middleManager1_hostname:8091\");\nbatch_workers.add(\"middleManager2_hostname:8091\");\nworkers = zkWorkers.keySet().toArray();\nvar sortedWorkers = new Array()\n;for(var i = 0; i < workers.length; i++){\n sortedWorkers[i] = workers[i];\n}\nArray.prototype.sort.call(sortedWorkers,function(a, b){return zkWorkers.get(b).getCurrCapacityUsed() - zkWorkers.get(a).getCurrCapa [...]
+ "function":"function (config, zkWorkers, task) {\nvar batch_workers = new java.util.ArrayList();\nbatch_workers.add(\"middleManager1_hostname:8091\");\nbatch_workers.add(\"middleManager2_hostname:8091\");\nworkers = zkWorkers.keySet().toArray();\nvar sortedWorkers = new Array()\n;for(var i = 0; i < workers.length; i++){\n sortedWorkers[i] = workers[i];\n}\nArray.prototype.sort.call(sortedWorkers,function(a, b){return zkWorkers.get(b).getCurrCapacityUsed() - zkWorkers.get(a).getCurrCapa [...]
}
```
@@ -1420,16 +1416,13 @@ Additional Peon configs include:
|`druid.peon.mode`|One of `local` or `remote`. Setting this property to `local` means you intend to run the Peon as a standalone process which is not recommended.|`remote`|
|`druid.indexer.task.baseDir`|Base temporary working directory.|`System.getProperty("java.io.tmpdir")`|
|`druid.indexer.task.baseTaskDir`|Base temporary working directory for tasks.|`${druid.indexer.task.baseDir}/persistent/task`|
-|`druid.indexer.task.defaultHadoopCoordinates`|Hadoop version to use with HadoopIndexTasks that do not request a particular version.|`org.apache.hadoop:hadoop-client-api:3.3.6`, `org.apache.hadoop:hadoop-client-runtime:3.3.6`|
|`druid.indexer.task.defaultRowFlushBoundary`|Highest row count before persisting to disk. Used for indexing generating tasks.|75000|
|`druid.indexer.task.directoryLockTimeout`|Wait this long for zombie Peons to exit before giving up on their replacements.|PT10M|
|`druid.indexer.task.gracefulShutdownTimeout`|Wait this long on Middle Manager restart for restorable tasks to gracefully exit.|PT5M|
-|`druid.indexer.task.hadoopWorkingPath`|Temporary working directory for Hadoop tasks.|`/tmp/druid-indexing`|
|`druid.indexer.task.restoreTasksOnRestart`|If true, Middle Managers will attempt to stop tasks gracefully on shutdown and restore them on restart.|false|
|`druid.indexer.task.ignoreTimestampSpecForDruidInputSource`|If true, tasks using the [Druid input source](../ingestion/input-sources.md) will ignore the provided timestampSpec, and will use the `__time` column of the input datasource. This option is provided for compatibility with ingestion specs written before Druid 0.22.0.|false|
|`druid.indexer.task.storeEmptyColumns`|Boolean value for whether or not to store empty columns during ingestion. When set to true, Druid stores every column specified in the [`dimensionsSpec`](../ingestion/ingestion-spec.md#dimensionsspec). If you use the string-based schemaless ingestion and don't specify any dimensions to ingest, you must also set [`includeAllDimensions`](../ingestion/ingestion-spec.md#dimensionsspec) for Druid to store empty columns.<br/><br/>If you set `storeEmptyCo [...]
|`druid.indexer.task.tmpStorageBytesPerTask`|Maximum number of bytes per task to be used to store temporary files on disk. This config is generally intended for internal usage. Attempts to set it are very likely to be overwritten by the TaskRunner that executes the task, so be sure of what you expect to happen before directly adjusting this configuration parameter. The config is documented here primarily to provide an understanding of what it means if/when someone sees that it has been s [...]
-|`druid.indexer.task.allowHadoopTaskExecution`|Conditional dictating if the cluster allows `index_hadoop` tasks to be executed. `index_hadoop` is deprecated, and defaulting to false will force cluster operators to acknowledge the deprecation and consciously opt in to using index_hadoop with the understanding that it will be removed in the future.|false|
|`druid.indexer.server.maxChatRequests`|Maximum number of concurrent requests served by a task's chat handler. Set to 0 to disable limiting.|0|
If the Peon is running in remote mode, there must be an Overlord up and running. Peons in remote mode can set the following configurations:
@@ -1484,9 +1477,7 @@ For most types of tasks, `SegmentWriteOutMediumFactory` can be configured per-ta
|`druid.worker.numConcurrentMerges`|Maximum number of segment persist or merge operations that can run concurrently across all tasks.|`druid.worker.capacity` / 2, rounded down|
|`druid.indexer.task.baseDir`|Base temporary working directory.|`System.getProperty("java.io.tmpdir")`|
|`druid.indexer.task.baseTaskDir`|Base temporary working directory for tasks.|`${druid.indexer.task.baseDir}/persistent/tasks`|
-|`druid.indexer.task.defaultHadoopCoordinates`|Hadoop version to use with HadoopIndexTasks that do not request a particular version.|`org.apache.hadoop:hadoop-client-api:3.3.6`, `org.apache.hadoop:hadoop-client-runtime:3.3.6`|
|`druid.indexer.task.gracefulShutdownTimeout`|Wait this long on Indexer restart for restorable tasks to gracefully exit.|`PT5M`|
-|`druid.indexer.task.hadoopWorkingPath`|Temporary working directory for Hadoop tasks.|`/tmp/druid-indexing`|
|`druid.indexer.task.restoreTasksOnRestart`|If true, the Indexer will attempt to stop tasks gracefully on shutdown and restore them on restart.|false|
|`druid.indexer.task.ignoreTimestampSpecForDruidInputSource`|If true, tasks using the [Druid input source](../ingestion/input-sources.md) will ignore the provided timestampSpec, and will use the `__time` column of the input datasource. This option is provided for compatibility with ingestion specs written before Druid 0.22.0.|false|
|`druid.indexer.task.storeEmptyColumns`|Boolean value for whether or not to store empty columns during ingestion. When set to true, Druid stores every column specified in the [`dimensionsSpec`](../ingestion/ingestion-spec.md#dimensionsspec). <br/><br/>If you set `storeEmptyColumns` to false, Druid SQL queries referencing empty columns will fail. If you intend to leave `storeEmptyColumns` disabled, you should either ingest placeholder data for empty columns or else not query on empty colu [...]
diff --git a/docs/design/segments.md b/docs/design/segments.md
index 6d2d9b5badb..7bf9c1b8862 100644
--- a/docs/design/segments.md
+++ b/docs/design/segments.md
@@ -34,7 +34,7 @@ changing the granularity of the segment time interval or partitioning your
data and/or adjusting the `targetRowsPerSegment` in your `partitionsSpec`.
A good starting point for this parameter is 5 million rows.
See the Sharding section below and the "Partitioning specification" section of
-the [Batch ingestion](../ingestion/hadoop.md#partitionsspec) documentation
+the [Batch ingestion](../ingestion/native-batch.md#partitionsspec) documentation
for more guidance.
## Segment file structure
diff --git a/docs/design/storage.md b/docs/design/storage.md
index 1854f549f37..cb237c84191 100644
--- a/docs/design/storage.md
+++ b/docs/design/storage.md
@@ -50,7 +50,7 @@ Indexing is the mechanism by which new segments are created, and handoff is the
On the indexing side:
1. An indexing task starts running and building a new segment. It must determine the identifier of the segment before it starts building it. For a task that is appending (like a Kafka task, or an index task in append mode) this is done by calling an "allocate" API on the Overlord to potentially add a new partition to an existing set of segments. For
-a task that is overwriting (like a Hadoop task, or an index task not in append mode) this is done by locking an interval and creating a new version number and new set of segments.
+a task that is overwriting (an index task not in append mode) this is done by locking an interval and creating a new version number and new set of segments.
2. If the indexing task is a realtime task (like a Kafka task) then the segment is immediately queryable at this point. It's available, but unpublished.
3. When the indexing task has finished reading data for the segment, it pushes it to deep storage and then publishes it by writing a record into the metadata store.
4. If the indexing task is a realtime task, then to ensure data is continuously available for queries, it waits for a Historical service to load the segment. If the indexing task is not a realtime task, it exits immediately.
@@ -116,18 +116,18 @@ On the ingestion side, Druid's primary [ingestion methods](../ingestion/index.md
- Supervised "seekable-stream" ingestion methods like [Kafka](../ingestion/kafka-ingestion.md) and [Kinesis](../ingestion/kinesis-ingestion.md). With these methods, Druid commits stream offsets to its [metadata store](metadata-storage.md) alongside segment metadata, in the same transaction. Note that ingestion of data that has not yet been published can be rolled back if ingestion tasks fail. In this case, partially-ingested data is
discarded, and Druid will resume ingestion from the last committed set of stream offsets. This ensures exactly-once publishing behavior.
-- [Hadoop-based batch ingestion](../ingestion/hadoop.md). Each task publishes all segment metadata in a single transaction.
+- [SQL REPLACE](../multi-stage-query/reference.md#replace) statements. The controller task publishes all segment metadata when the statement is finished executing.
- [Native batch ingestion](../ingestion/native-batch.md). In parallel mode, the supervisor task publishes all segment metadata in a single transaction after the subtasks are finished. In simple (single-task) mode, the single task publishes all segment metadata in a single transaction after it is complete.
Additionally, some ingestion methods offer an _idempotency_ guarantee. This means that repeated executions of the same ingestion will not cause duplicate data to be ingested:
- Supervised "seekable-stream" ingestion methods like [Kafka](../ingestion/kafka-ingestion.md) and [Kinesis](../ingestion/kinesis-ingestion.md) are idempotent due to the fact that stream offsets and segment metadata are stored together and updated in lock-step.
-- [Hadoop-based batch ingestion](../ingestion/hadoop.md) is idempotent unless one of your input sources is the same Druid datasource that you are ingesting into. In this case, running the same task twice is non-idempotent, because you are adding to existing data instead of overwriting it.
+- [SQL REPLACE](../multi-stage-query/reference.md#replace) statements are idempotent unless reading from the same Druid datasource that you are ingesting into. In this case, running the same task twice is non-idempotent, because you are adding to existing data instead of overwriting it.
- [Native batch ingestion](../ingestion/native-batch.md) is idempotent unless
[`appendToExisting`](../ingestion/native-batch.md) is true, or one of your input sources is the same Druid datasource that you are ingesting into. In either of these two cases, running the same task twice is non-idempotent, because you are adding to existing data instead of overwriting it.
On the query side, the Druid Broker is responsible for ensuring that a consistent set of segments is involved in a given query. It selects the appropriate set of segment versions to use when the query starts based on what is currently available. This is supported by atomic replacement, a feature that ensures that from a user's perspective, queries flip instantaneously from an older version of data to a newer set of data, with no consistency or performance impact.
-This is used for Hadoop-based batch ingestion, native batch ingestion when `appendToExisting` is false, and compaction.
+This is used for SQL REPLACE statements, native batch ingestion when `appendToExisting` is false, and compaction.
Note that atomic replacement happens for each time chunk individually. If a batch ingestion task or compaction involves multiple time chunks, then each time chunk will undergo atomic replacement soon after the task finishes, but the replacements will not all happen simultaneously.
diff --git a/docs/development/extensions-contrib/delta-lake.md b/docs/development/extensions-contrib/delta-lake.md
index 88f3a2c77f3..1d8cc801831 100644
--- a/docs/development/extensions-contrib/delta-lake.md
+++ b/docs/development/extensions-contrib/delta-lake.md
@@ -45,9 +45,7 @@ Druid version:
java \
-cp "lib/*" \
-Ddruid.extensions.directory="extensions" \
- -Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" \
org.apache.druid.cli.Main tools pull-deps \
- --no-default-hadoop \
-c "org.apache.druid.extensions.contrib:druid-deltalake-extensions:<VERSION>"
```
diff --git a/docs/development/extensions-contrib/iceberg.md b/docs/development/extensions-contrib/iceberg.md
index c2652cc7858..d895093691a 100644
--- a/docs/development/extensions-contrib/iceberg.md
+++ b/docs/development/extensions-contrib/iceberg.md
@@ -131,9 +131,7 @@ Druid version:
java \
-cp "lib/*" \
-Ddruid.extensions.directory="extensions" \
- -Ddruid.extensions.hadoopDependenciesDir="hadoop-dependencies" \
org.apache.druid.cli.Main tools pull-deps \
- --no-default-hadoop \
-c "org.apache.druid.extensions.contrib:druid-iceberg-extensions:<VERSION>"
```
diff --git a/docs/development/extensions-contrib/materialized-view.md b/docs/development/extensions-contrib/materialized-view.md
deleted file mode 100644
index a493c3c4177..00000000000
--- a/docs/development/extensions-contrib/materialized-view.md
+++ /dev/null
@@ -1,136 +0,0 @@
----
-id: materialized-view
-title: "Materialized View"
----
-
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-
-To use this Apache Druid feature, make sure to load `materialized-view-selection` and `materialized-view-maintenance`. In addition, this feature currently requires a Hadoop cluster.
-
-This feature enables Druid to greatly improve the query performance, especially when the query dataSource has a very large number of dimensions but the query only required several dimensions. This feature includes two parts. One is `materialized-view-maintenance`, and the other is `materialized-view-selection`.
-
-## Materialized-view-maintenance
-In materialized-view-maintenance, dataSources user ingested are called "base-dataSource". For each base-dataSource, we can submit `derivativeDataSource` supervisors to create and maintain other dataSources which we called "derived-dataSource". The dimensions and metrics of derived-dataSources are the subset of base-dataSource's.
-The `derivativeDataSource` supervisor is used to keep the timeline of derived-dataSource consistent with base-dataSource. Each `derivativeDataSource` supervisor is responsible for one derived-dataSource.
-
-A sample derivativeDataSource supervisor spec is shown below:
-
-```json
- {
- "type": "derivativeDataSource",
- "baseDataSource": "wikiticker",
- "dimensionsSpec": {
- "dimensions": [
- "isUnpatrolled",
- "metroCode",
- "namespace",
- "page",
- "regionIsoCode",
- "regionName",
- "user"
- ]
- },
- "metricsSpec": [
- {
- "name": "count",
- "type": "count"
- },
- {
- "name": "added",
- "type": "longSum",
- "fieldName": "added"
- }
- ],
- "tuningConfig": {
- "type": "hadoop"
- }
- }
-```
-
-**Supervisor Configuration**
-
-|Field|Description|Required|
-|--------|-----------|---------|
-|Type |The supervisor type. This should always be `derivativeDataSource`.|yes|
-|baseDataSource |The name of base dataSource. This dataSource data should be already stored inside Druid, and the dataSource will be used as input data.|yes|
-|dimensionsSpec |Specifies the dimensions of the data. These dimensions must be the subset of baseDataSource's dimensions.|yes|
-|metricsSpec |A list of aggregators. These metrics must be the subset of baseDataSource's metrics. See [aggregations](../../querying/aggregations.md).|yes|
-|tuningConfig |TuningConfig must be HadoopTuningConfig. See [Hadoop tuning config](../../ingestion/hadoop.md#tuningconfig).|yes|
-|dataSource |The name of this derived dataSource. |no(default=baseDataSource-hashCode of supervisor)|
-|hadoopDependencyCoordinates |A JSON array of Hadoop dependency coordinates that Druid will use, this property will override the default Hadoop coordinates. Once specified, Druid will look for those Hadoop dependencies from the location specified by druid.extensions.hadoopDependenciesDir |no|
-|classpathPrefix |Classpath that will be prepended for the Peon process. |no|
-|context |See below. |no|
-
-**Context**
-
-|Field|Description|Required|
-|--------|-----------|---------|
-|maxTaskCount |The max number of tasks the supervisor can submit simultaneously. |no(default=1)|
-
-## Materialized-view-selection
-
-In materialized-view-selection, we implement a new query type `view`. When we request a view query, Druid will try its best to optimize the query based on query dataSource and intervals.
-
-A sample view query spec is shown below:
-
-```json
- {
- "queryType": "view",
- "query": {
- "queryType": "groupBy",
- "dataSource": "wikiticker",
- "granularity": "all",
- "dimensions": [
- "user"
- ],
- "limitSpec": {
- "type": "default",
- "limit": 1,
- "columns": [
- {
- "dimension": "added",
- "direction": "descending",
- "dimensionOrder": "numeric"
- }
- ]
- },
- "aggregations": [
- {
- "type": "longSum",
- "name": "added",
- "fieldName": "added"
- }
- ],
- "intervals": [
- "2015-09-12/2015-09-13"
- ]
- }
- }
-```
-
-There are 2 parts in a view query:
-
-|Field|Description|Required|
-|--------|-----------|---------|
-|queryType |The query type. This should always be view |yes|
-|query |The real query of this `view` query. The real query must be [groupBy](../../querying/groupbyquery.md), [topN](../../querying/topnquery.md), or [timeseries](../../querying/timeseriesquery.md) type.|yes|
-
-**Note that Materialized View is currently designated as experimental. Please make sure the time of all processes are the same and increase monotonically. Otherwise, some unexpected errors may happen on query results.**
diff --git a/docs/development/extensions-contrib/thrift.md b/docs/development/extensions-contrib/thrift.md
index 8e15c154b72..0526ae567a1 100644
--- a/docs/development/extensions-contrib/thrift.md
+++ b/docs/development/extensions-contrib/thrift.md
@@ -25,7 +25,7 @@ title: "Thrift"
To use this Apache Druid extension, [include](../../configuration/extensions.md#loading-extensions) `druid-thrift-extensions` in the extensions load list.
-This extension enables Druid to ingest Thrift-encoded data from streaming sources such as Kafka and Kinesis, as well as from Hadoop batch jobs reading SequenceFile or LzoThriftBlock files. The binary, compact, and JSON Thrift wire protocols are all supported, with optional Base64 encoding.
+This extension enables Druid to ingest Thrift-encoded data from streaming sources such as Kafka and Kinesis.
You may want to use another version of thrift, change the dependency in pom and compile yourself.
@@ -117,57 +117,3 @@ The following Kafka supervisor spec ingests compact-encoded `Book` messages, usi
}
```
-## LZO Support
-
-If you plan to read LZO-compressed Thrift files, you will need to download version 0.4.19 of the [hadoop-lzo JAR](https://mvnrepository.com/artifact/com.hadoop.gplcompression/hadoop-lzo/0.4.19) and place it in your `extensions/druid-thrift-extensions` directory.
-
-## Thrift parser (deprecated)
-
-`ThriftInputRowParser` is the legacy parser-based approach to Thrift ingestion. It is deprecated in favor of `ThriftInputFormat` above and will be removed in a future release.
-
-| Field | Type | Description | Required |
-|-------|------|-------------|----------|
-| type | String | Must be `thrift` | yes |
-| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data. Should be a JSON parseSpec. | yes |
-| thriftJar | String | Path to the Thrift JAR. If not provided, the class is looked up from the classpath. For Hadoop batch ingestion the JAR should be uploaded to HDFS first and `jobProperties` configured with `"tmpjars":"/path/to/your/thrift.jar"`. | no |
-| thriftClass | String | Fully qualified class name of the Thrift-generated class. | yes |
-
-Batch ingestion example using the HadoopDruidIndexer. The `inputFormat` of `inputSpec` in `ioConfig` can be either `"org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat"` or `"com.twitter.elephantbird.mapreduce.input.LzoThriftBlockInputFormat"`. When using `LzoThriftBlockInputFormat`, the Thrift class must be provided twice.
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "dataSchema": {
- "dataSource": "book",
- "parser": {
- "type": "thrift",
- "jarPath": "book.jar",
- "thriftClass": "org.apache.druid.data.input.thrift.Book",
- "protocol": "compact",
- "parseSpec": {
- "format": "json",
- "timestampSpec": {},
- "dimensionsSpec": {}
- }
- },
- "metricsSpec": [],
- "granularitySpec": {}
- },
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat",
- "paths": "/user/to/some/book.seq"
- }
- },
- "tuningConfig": {
- "type": "hadoop",
- "jobProperties": {
- "tmpjars": "/user/h_user_profile/du00/druid/test/book.jar"
- }
- }
- }
-}
-```
diff --git a/docs/development/extensions-core/avro.md b/docs/development/extensions-core/avro.md
index 2ce6451a422..3dbf9beb7de 100644
--- a/docs/development/extensions-core/avro.md
+++ b/docs/development/extensions-core/avro.md
@@ -25,7 +25,6 @@ title: "Apache Avro"
This Apache Druid extension enables Druid to ingest and parse the Apache Avro data format as follows:
- [Avro stream input format](../../ingestion/data-formats.md#avro-stream) for Kafka and Kinesis.
- [Avro OCF input format](../../ingestion/data-formats.md#avro-ocf) for native batch ingestion.
-- [Avro Hadoop Parser](../../ingestion/data-formats.md#avro-hadoop-parser).
## Load the Avro extension
diff --git a/docs/development/extensions-core/google.md b/docs/development/extensions-core/google.md
index 6df933f2da6..a9d1edefec4 100644
--- a/docs/development/extensions-core/google.md
+++ b/docs/development/extensions-core/google.md
@@ -37,8 +37,7 @@ To configure connectivity to google cloud, run druid processes with `GOOGLE_APPL
### Reading data from Google Cloud Storage
The [Google Cloud Storage input source](../../ingestion/input-sources.md) is supported by the [Parallel task](../../ingestion/native-batch.md)
-to read objects directly from Google Cloud Storage. If you use the [Hadoop task](../../ingestion/hadoop.md),
-you can read data from Google Cloud Storage by specifying the paths in your [`inputSpec`](../../ingestion/hadoop.md#inputspec).
+to read objects directly from Google Cloud Storage.
### Deep Storage
diff --git a/docs/development/extensions-core/hdfs.md b/docs/development/extensions-core/hdfs.md
index b1d2d0ceaab..f6994e66310 100644
--- a/docs/development/extensions-core/hdfs.md
+++ b/docs/development/extensions-core/hdfs.md
@@ -38,129 +38,4 @@ To use this Apache Druid extension, [include](../../configuration/extensions.md#
Besides the above settings, you also need to include all Hadoop configuration files (such as `core-site.xml`, `hdfs-site.xml`)
in the Druid classpath. One way to do this is copying all those files under `${DRUID_HOME}/conf/_common`.
-If you are using the Hadoop ingestion, set your output directory to be a location on Hadoop and it will work.
If you want to eagerly authenticate against a secured hadoop/hdfs cluster you must set `druid.hadoop.security.kerberos.principal` and `druid.hadoop.security.kerberos.keytab`, this is an alternative to the cron job method that runs `kinit` command periodically.
-
-### Configuration for Cloud Storage
-
-You can also use the Amazon S3 or the Google Cloud Storage as the deep storage via HDFS.
-
-#### Configuration for Amazon S3
-
-To use the Amazon S3 as the deep storage, you need to configure `druid.storage.storageDirectory` properly.
-
-|Property|Possible Values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.storage.type`|hdfs| |Must be set.|
-|`druid.storage.storageDirectory`|s3a://bucket/example/directory or s3n://bucket/example/directory|Path to the deep storage|Must be set.|
-
-You also need to include the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/), especially the `hadoop-aws.jar` in the Druid classpath.
-Run the below command to install the `hadoop-aws.jar` file under `${DRUID_HOME}/extensions/druid-hdfs-storage` in all nodes.
-
-```bash
-${DRUID_HOME}/bin/run-java -classpath "${DRUID_HOME}/lib/*" org.apache.druid.cli.Main tools pull-deps -h "org.apache.hadoop:hadoop-aws:${HADOOP_VERSION}";
-cp ${DRUID_HOME}/hadoop-dependencies/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar ${DRUID_HOME}/extensions/druid-hdfs-storage/
-```
-
-Finally, you need to add the below properties in the `core-site.xml`.
-For more configurations, see the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/).
-
-```xml
-<property>
- <name>fs.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
- <description>The implementation class of the S3A Filesystem</description>
-</property>
-
-<property>
- <name>fs.AbstractFileSystem.s3a.impl</name>
- <value>org.apache.hadoop.fs.s3a.S3A</value>
- <description>The implementation class of the S3A AbstractFileSystem.</description>
-</property>
-
-<property>
- <name>fs.s3a.access.key</name>
- <description>AWS access key ID. Omit for IAM role-based or provider-based authentication.</description>
- <value>your access key</value>
-</property>
-
-<property>
- <name>fs.s3a.secret.key</name>
- <description>AWS secret key. Omit for IAM role-based or provider-based authentication.</description>
- <value>your secret key</value>
-</property>
-```
-
-#### Configuration for Google Cloud Storage
-
-To use the Google Cloud Storage as the deep storage, you need to configure `druid.storage.storageDirectory` properly.
-
-|Property|Possible Values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.storage.type`|hdfs||Must be set.|
-|`druid.storage.storageDirectory`|gs://bucket/example/directory|Path to the deep storage|Must be set.|
-
-All services that need to access GCS need to have the [GCS connector jar](https://cloud.google.com/dataproc/docs/concepts/connectors/cloud-storage#other_sparkhadoop_clusters) in their class path.
-Please read the [install instructions](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md)
-to properly set up the necessary libraries and configurations.
-One option is to place this jar in `${DRUID_HOME}/lib/` and `${DRUID_HOME}/extensions/druid-hdfs-storage/`.
-
-Finally, you need to configure the `core-site.xml` file with the filesystem
-and authentication properties needed for GCS. You may want to copy the below
-example properties. Please follow the instructions at
-[https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md)
-for more details.
-For more configurations, [GCS core default](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.0.0/gcs/conf/gcs-core-default.xml)
-and [GCS core template](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/src/test/resources/core-site.xml).
-
-```xml
-<property>
- <name>fs.gs.impl</name>
- <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem</value>
- <description>The FileSystem for gs: (GCS) uris.</description>
-</property>
-
-<property>
- <name>fs.AbstractFileSystem.gs.impl</name>
- <value>com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS</value>
- <description>The AbstractFileSystem for gs: uris.</description>
-</property>
-
-<property>
- <name>google.cloud.auth.service.account.enable</name>
- <value>true</value>
- <description>
- Whether to use a service account for GCS authorization.
- Setting this property to `false` will disable use of service accounts for
- authentication.
- </description>
-</property>
-
-<property>
- <name>google.cloud.auth.service.account.json.keyfile</name>
- <value>/path/to/keyfile</value>
- <description>
- The JSON key file of the service account used for GCS
- access when google.cloud.auth.service.account.enable is true.
- </description>
-</property>
-```
-
-## Reading data from HDFS or Cloud Storage
-
-### Native batch ingestion
-
-The [HDFS input source](../../ingestion/input-sources.md#hdfs-input-source) is supported by the [Parallel task](../../ingestion/native-batch.md)
-to read files directly from the HDFS Storage. You may be able to read objects from cloud storage
-with the HDFS input source, but we highly recommend to use a proper
-[Input Source](../../ingestion/input-sources.md) instead if possible because
-it is simple to set up. For now, only the [S3 input source](../../ingestion/input-sources.md#s3-input-source)
-and the [Google Cloud Storage input source](../../ingestion/input-sources.md#google-cloud-storage-input-source)
-are supported for cloud storage types, and so you may still want to use the HDFS input source
-to read from cloud storage other than those two.
-
-### Hadoop-based ingestion
-
-If you use the [Hadoop ingestion](../../ingestion/hadoop.md), you can read data from HDFS
-by specifying the paths in your [`inputSpec`](../../ingestion/hadoop.md#inputspec).
-See the [Static](../../ingestion/hadoop.md#static) inputSpec for details.
diff --git a/docs/development/extensions-core/orc.md b/docs/development/extensions-core/orc.md
index 4be58674099..cd912515526 100644
--- a/docs/development/extensions-core/orc.md
+++ b/docs/development/extensions-core/orc.md
@@ -26,59 +26,8 @@ title: "ORC Extension"
This Apache Druid extension enables Druid to ingest and understand the Apache ORC data format.
-The extension provides the [ORC input format](../../ingestion/data-formats.md#orc) and the [ORC Hadoop parser](../../ingestion/data-formats.md#orc-hadoop-parser)
-for [native batch ingestion](../../ingestion/native-batch.md) and [Hadoop batch ingestion](../../ingestion/hadoop.md), respectively.
-Please see corresponding docs for details.
+The extension provides the [ORC input format](../../ingestion/data-formats.md#orc)
+for [native batch ingestion](../../ingestion/native-batch.md).
To use this extension, make sure to [include](../../configuration/extensions.md#loading-extensions) `druid-orc-extensions` in the extensions load list.
-### Migration from 'contrib' extension
-This extension, first available in version 0.15.0, replaces the previous 'contrib' extension which was available until
-0.14.0-incubating. While this extension can index any data the 'contrib' extension could, the JSON spec for the
-ingestion task is *incompatible*, and will need modified to work with the newer 'core' extension.
-
-To migrate to 0.15.0+:
-
-* In `inputSpec` of `ioConfig`, `inputFormat` must be changed from `"org.apache.hadoop.hive.ql.io.orc.OrcNewInputFormat"` to
-`"org.apache.orc.mapreduce.OrcInputFormat"`
-* The 'contrib' extension supported a `typeString` property, which provided the schema of the
-ORC file, of which was essentially required to have the types correct, but notably _not_ the column names, which
-facilitated column renaming. In the 'core' extension, column renaming can be achieved with
-[`flattenSpec`](../../ingestion/ingestion-spec.md#flattenspec). For example, `"typeString":"struct<time:string,name:string>"`
-with the actual schema `struct<_col0:string,_col1:string>`, to preserve Druid schema would need replaced with:
-
-```json
-"flattenSpec": {
- "fields": [
- {
- "type": "path",
- "name": "time",
- "expr": "$._col0"
- },
- {
- "type": "path",
- "name": "name",
- "expr": "$._col1"
- }
- ]
- ...
-}
-```
-
-* The 'contrib' extension supported a `mapFieldNameFormat` property, which provided a way to specify a dimension to
- flatten `OrcMap` columns with primitive types. This functionality has also been replaced with
- [`flattenSpec`](../../ingestion/ingestion-spec.md#flattenspec). For example: `"mapFieldNameFormat": "<PARENT>_<CHILD>"`
- for a dimension `nestedData_dim1`, to preserve Druid schema could be replaced with
-
- ```json
-"flattenSpec": {
- "fields": [
- {
- "type": "path",
- "name": "nestedData_dim1",
- "expr": "$.nestedData.dim1"
- }
- ]
- ...
-}
-```
diff --git a/docs/development/extensions-core/parquet.md b/docs/development/extensions-core/parquet.md
index a655c8989c8..cbeb446b82c 100644
--- a/docs/development/extensions-core/parquet.md
+++ b/docs/development/extensions-core/parquet.md
@@ -23,14 +23,5 @@ title: "Apache Parquet Extension"
-->
-This Apache Druid module extends [Druid Hadoop based indexing](../../ingestion/hadoop.md) to ingest data directly from offline
+This Apache Druid module extends [Druid native batch](../../ingestion/native-batch.md) to ingest data directly from offline
Apache Parquet files.
-
-Note: If using the `parquet-avro` parser for Apache Hadoop based indexing, `druid-parquet-extensions` depends on the `druid-avro-extensions` module, so be sure to
- [include both](../../configuration/extensions.md#loading-extensions).
-
-The `druid-parquet-extensions` provides the [Parquet input format](../../ingestion/data-formats.md#parquet), the [Parquet Hadoop parser](../../ingestion/data-formats.md#parquet-hadoop-parser),
-and the [Parquet Avro Hadoop Parser](../../ingestion/data-formats.md#parquet-avro-hadoop-parser) with `druid-avro-extensions`.
-The Parquet input format is available for [native batch ingestion](../../ingestion/native-batch.md)
-and the other 2 parsers are for [Hadoop batch ingestion](../../ingestion/hadoop.md).
-Please see corresponding docs for details.
diff --git a/docs/development/extensions-core/s3.md b/docs/development/extensions-core/s3.md
index ed33d4337e0..ae8f0a9f564 100644
--- a/docs/development/extensions-core/s3.md
+++ b/docs/development/extensions-core/s3.md
@@ -35,9 +35,6 @@ To use this Apache Druid extension, [include](../../configuration/extensions.md#
Use a native batch [Parallel task](../../ingestion/native-batch.md) with an [S3 input source](../../ingestion/input-sources.md#s3-input-source) to read objects directly from S3.
-Alternatively, use a [Hadoop task](../../ingestion/hadoop.md),
-and specify S3 paths in your [`inputSpec`](../../ingestion/hadoop.md#inputspec).
-
To read objects from S3, you must supply [connection information](#configuration) in configuration.
### Deep Storage
@@ -56,7 +53,6 @@ To use S3 for Deep Storage, you must supply [connection information](#configurat
|`druid.storage.baseKey`|A prefix string that will be prepended to the object names for the segments published to S3 deep storage|Must be set.|
|`druid.storage.type`|Global deep storage provider. Must be set to `s3` to make use of this extension.|Must be set (likely `s3`).|
|`druid.storage.disableAcl`|Boolean flag for how object permissions are handled. To use ACLs, set this property to `false`. To use Object Ownership, set it to `true`. The permission requirements for ACLs and Object Ownership are different. For more information, see [S3 permissions settings](#s3-permissions-settings).|false|
-|`druid.storage.useS3aSchema`|If true, use the "s3a" filesystem when using Hadoop-based ingestion. If false, the "s3n" filesystem will be used. Only affects Hadoop-based ingestion.|false|
|`druid.storage.zip`|`true`, `false`|Whether segments in `s3` are written as directories (`false`) or zip files (`true`).|`false`|
|`druid.storage.transfer.useTransferManager`| If true, use AWS S3 Transfer Manager to upload segments to S3.|true|
|`druid.storage.transfer.minimumUploadPartSize`| Minimum size (in bytes) of each part in a multipart upload.|20971520 (20 MB)|
diff --git a/docs/development/overview.md b/docs/development/overview.md
index 11c67ddfa2c..b51c4f8c01a 100644
--- a/docs/development/overview.md
+++ b/docs/development/overview.md
@@ -65,13 +65,6 @@ as data analysis and partitioning depending on the task specification. Segment g
`SinglePhaseSubTask.java`, `PartialHashSegmentGenerateTask.java`, or `PartialRangeSegmentGenerateTask.java` through
`BatchAppenderator`, and the persist and hand-off logic is in `BatchAppenderatorDriver.java`.
-## Hadoop-based Batch Ingestion
-
-The two main Hadoop indexing classes are `HadoopDruidDetermineConfigurationJob.java` for the job to determine how many Druid
-segments to create, and `HadoopDruidIndexerJob.java`, which creates Druid segments.
-
-At some point in the future, we may move the Hadoop ingestion code out of core Druid.
-
## Internal UIs
Druid currently has two internal UIs. One is for the Coordinator and one is for the Overlord.
diff --git a/docs/ingestion/data-formats.md b/docs/ingestion/data-formats.md
index d6c588e2184..c238117ea49 100644
--- a/docs/ingestion/data-formats.md
+++ b/docs/ingestion/data-formats.md
@@ -81,9 +81,6 @@ You can use the `inputFormat` field to specify the data format for your input da
`inputFormat` doesn't support all data formats or ingestion methods supported by Druid.
:::
-Especially if you want to use the Hadoop ingestion, you still need to use the [Parser](#parser).
-If your data is formatted in some format not listed in this section, please consider using the Parser instead.
-
All forms of Druid ingestion require some form of schema object. The format of the data to be ingested is specified using the `inputFormat` entry in your [`ioConfig`](ingestion-spec.md#ioconfig).
### JSON
@@ -193,9 +190,6 @@ For example:
### ORC
To use the ORC input format, load the Druid Orc extension ( [`druid-orc-extensions`](../development/extensions-core/orc.md)).
-:::info
- To upgrade from versions earlier than 0.15.0 to 0.15.0 or new, read [Migration from 'contrib' extension](../development/extensions-core/orc.md#migration-from-contrib-extension).
-:::
Configure the ORC `inputFormat` to load ORC data as follows:
@@ -977,578 +971,9 @@ Each line can be further parsed using [`parseSpec`](#parsespec).
| Field | Type | Description | Required |
|-------|------|-------------|----------|
-| type | String | Set value to `string` for most cases. Otherwise use `hadoopyString` for Hadoop indexing. | yes |
+| type | String | Set value to `string` for most cases. | yes |
| parseSpec | JSON Object | Specifies the format, timestamp, and dimensions of the data. | yes |
-### Avro Hadoop Parser
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-:::
-
-
-:::info
-You need to include [`druid-avro-extensions`](../development/extensions-core/avro.md) as an extension to use the Avro Hadoop Parser.
-
- See the [Avro Types](../development/extensions-core/avro.md#avro-types) section for how Avro types are handled in Druid
-:::
-
-This parser is for [Hadoop batch ingestion](./hadoop.md).
-The `inputFormat` of `inputSpec` in `ioConfig` must be set to `"org.apache.druid.data.input.avro.AvroValueInputFormat"`.
-You may want to set Avro reader's schema in `jobProperties` in `tuningConfig`,
-e.g.: `"avro.schema.input.value.path": "/path/to/your/schema.avsc"` or
-`"avro.schema.input.value": "your_schema_JSON_object"`.
-If the Avro reader's schema is not set, the schema in Avro object container file will be used.
-See [Avro specification](http://avro.apache.org/docs/1.7.7/spec.html#Schema+Resolution) for more information.
-
-| Field | Type | Description | Required |
-|-------|------|-------------|----------|
-| type | String | Set value to `avro_hadoop`. | yes |
-| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data. Should be an "avro" parseSpec. | yes |
-| fromPigAvroStorage | Boolean | Specifies whether the data file is stored using AvroStorage. | no(default == false) |
-
-An Avro parseSpec can contain a [`flattenSpec`](#flattenspec) using either the "root" or "path"
-field types, which can be used to read nested Avro records. The "jq" and "tree" field type is not currently supported
-for Avro.
-
-For example, using Avro Hadoop parser with custom reader's schema file:
-
-```json
-{
- "type" : "index_hadoop",
- "spec" : {
- "dataSchema" : {
- "dataSource" : "",
- "parser" : {
- "type" : "avro_hadoop",
- "parseSpec" : {
- "format": "avro",
- "timestampSpec": <standard timestampSpec>,
- "dimensionsSpec": <standard dimensionsSpec>,
- "flattenSpec": <optional>
- }
- }
- },
- "ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "static",
- "inputFormat": "org.apache.druid.data.input.avro.AvroValueInputFormat",
- "paths" : ""
- }
- },
- "tuningConfig" : {
- "jobProperties" : {
- "avro.schema.input.value.path" : "/path/to/my/schema.avsc"
- }
- }
- }
-}
-```
-
-### ORC Hadoop Parser
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-
-:::
-
-
-:::info
- You need to include the [`druid-orc-extensions`](../development/extensions-core/orc.md) as an extension to use the ORC Hadoop Parser.
-:::
-
-:::info
- If you are considering upgrading from earlier than 0.15.0 to 0.15.0 or a higher version,
- please read [Migration from 'contrib' extension](../development/extensions-core/orc.md#migration-from-contrib-extension) carefully.
-:::
-
-This parser is for [Hadoop batch ingestion](./hadoop.md).
-The `inputFormat` of `inputSpec` in `ioConfig` must be set to `"org.apache.orc.mapreduce.OrcInputFormat"`.
-
-|Field | Type | Description | Required|
-|----------|-------------|----------------------------------------------------------------------------------------|---------|
-| type | String | Set value to `orc`. | yes |
-|parseSpec | JSON Object | Specifies the timestamp and dimensions of the data (`timeAndDims` and `orc` format) and a `flattenSpec` (`orc` format). | yes|
-
-The parser supports two `parseSpec` formats: `orc` and `timeAndDims`.
-
-`orc` supports auto field discovery and flattening, if specified with a [`flattenSpec`](#flattenspec).
-If no `flattenSpec` is specified, `useFieldDiscovery` will be enabled by default. Specifying a `dimensionSpec` is
-optional if `useFieldDiscovery` is enabled: if a `dimensionSpec` is supplied, the list of `dimensions` it defines will be
-the set of ingested dimensions, if missing the discovered fields will make up the list.
-
-`timeAndDims` parse spec must specify which fields will be extracted as dimensions through the `dimensionSpec`.
-
-[All column types](https://orc.apache.org/docs/types.html) are supported, with the exception of `union` types. Columns of
- `list` type, if filled with primitives, may be used as a multi-value dimension, or specific elements can be extracted with
-`flattenSpec` expressions. Likewise, primitive fields may be extracted from `map` and `struct` types in the same manner.
-Auto field discovery will automatically create a string dimension for every (non-timestamp) primitive or `list` of
-primitives, as well as any flatten expressions defined in the `flattenSpec`.
-
-#### Hadoop job properties
-
-Like most Hadoop jobs, the best outcomes will add `"mapreduce.job.user.classpath.first": "true"` or
-`"mapreduce.job.classloader": "true"` to the `jobProperties` section of `tuningConfig`. Note that it is likely if using
-`"mapreduce.job.classloader": "true"` that you will need to set `mapreduce.job.classloader.system.classes` to include
-`-org.apache.hadoop.hive.` to instruct Hadoop to load `org.apache.hadoop.hive` classes from the application jars instead
-of system jars, e.g.
-
-```json
-...
- "mapreduce.job.classloader": "true",
- "mapreduce.job.classloader.system.classes" : "java., javax.accessibility., javax.activation., javax.activity., javax.annotation., javax.annotation.processing., javax.crypto., javax.imageio., javax.jws., javax.lang.model., -javax.management.j2ee., javax.management., javax.naming., javax.net., javax.print., javax.rmi., javax.script., -javax.security.auth.message., javax.security.auth., javax.security.cert., javax.security.sasl., javax.sound., javax.sql., javax.swing., javax.tools., jav [...]
-...
-```
-
-This is due to the `hive-storage-api` dependency of the
-`orc-mapreduce` library, which provides some classes under the `org.apache.hadoop.hive` package. If instead using the
-setting `"mapreduce.job.user.classpath.first": "true"`, then this will not be an issue.
-
-#### Examples
-
-##### `orc` parser, `orc` parseSpec, auto field discovery, flatten expressions
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",
- "paths": "path/to/file.orc"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "orc",
- "parseSpec": {
- "format": "orc",
- "flattenSpec": {
- "useFieldDiscovery": true,
- "fields": [
- {
- "type": "path",
- "name": "nestedDim",
- "expr": "$.nestedData.dim1"
- },
- {
- "type": "path",
- "name": "listDimFirstItem",
- "expr": "$.listDim[1]"
- }
- ]
- },
- "timestampSpec": {
- "column": "timestamp",
- "format": "millis"
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
- }
-}
-```
-
-##### `orc` parser, `orc` parseSpec, field discovery with no flattenSpec or dimensionSpec
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",
- "paths": "path/to/file.orc"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "orc",
- "parseSpec": {
- "format": "orc",
- "timestampSpec": {
- "column": "timestamp",
- "format": "millis"
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
- }
-}
-```
-
-##### `orc` parser, `orc` parseSpec, no autodiscovery
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",
- "paths": "path/to/file.orc"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "orc",
- "parseSpec": {
- "format": "orc",
- "flattenSpec": {
- "useFieldDiscovery": false,
- "fields": [
- {
- "type": "path",
- "name": "nestedDim",
- "expr": "$.nestedData.dim1"
- },
- {
- "type": "path",
- "name": "listDimFirstItem",
- "expr": "$.listDim[1]"
- }
- ]
- },
- "timestampSpec": {
- "column": "timestamp",
- "format": "millis"
- },
- "dimensionsSpec": {
- "dimensions": [
- "dim1",
- "dim3",
- "nestedDim",
- "listDimFirstItem"
- ],
- "dimensionExclusions": [],
- "spatialDimensions": []
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
- }
-}
-```
-
-##### `orc` parser, `timeAndDims` parseSpec
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.orc.mapreduce.OrcInputFormat",
- "paths": "path/to/file.orc"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "orc",
- "parseSpec": {
- "format": "timeAndDims",
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [
- "dim1",
- "dim2",
- "dim3",
- "listDim"
- ],
- "dimensionExclusions": [],
- "spatialDimensions": []
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
-}
-
-```
-
-### Parquet Hadoop Parser
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-:::
-
-
-:::info
- You need to include the [`druid-parquet-extensions`](../development/extensions-core/parquet.md) as an extension to use the Parquet Hadoop Parser.
-:::
-
-The Parquet Hadoop parser is for [Hadoop batch ingestion](./hadoop.md) and parses Parquet files directly.
-The `inputFormat` of `inputSpec` in `ioConfig` must be set to `org.apache.druid.data.input.parquet.DruidParquetInputFormat`.
-
-The Parquet Hadoop Parser supports auto field discovery and flattening if provided with a
-[`flattenSpec`](#flattenspec) with the `parquet` `parseSpec`. Parquet nested list and map
-[logical types](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) _should_ operate correctly with
-JSON path expressions for all supported types.
-
-|Field | Type | Description | Required|
-|----------|-------------|----------------------------------------------------------------------------------------|---------|
-| type | String | Set value to `parquet`. | yes |
-| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data, and optionally, a flatten spec. Valid parseSpec formats are `timeAndDims` and `parquet`. | yes |
-| binaryAsString | Boolean | Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string. | no(default = false) |
-
-When the time dimension is a [DateType column](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md),
-a format should not be supplied. When the format is UTF8 (String), either `auto` or a explicitly defined
-[format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat) is required.
-
-#### Parquet Hadoop Parser vs Parquet Avro Hadoop Parser
-
-Both parsers read from Parquet files, but slightly differently. The main
-differences are:
-
-- The Parquet Hadoop Parser uses a simple conversion while the Parquet Avro Hadoop Parser
-converts Parquet data into avro records first with the `parquet-avro` library and then
-parses avro data using the `druid-avro-extensions` module to ingest into Druid.
-- The Parquet Hadoop Parser sets a hadoop job property
-`parquet.avro.add-list-element-records` to `false` (which normally defaults to `true`), in order to 'unwrap' primitive
-list elements into multi-value dimensions.
-- The Parquet Hadoop Parser supports `int96` Parquet values, while the Parquet Avro Hadoop Parser does not.
-There may also be some subtle differences in the behavior of JSON path expression evaluation of `flattenSpec`.
-
-Based on those differences, we suggest using the Parquet Hadoop Parser over the Parquet Avro Hadoop Parser
-to allow ingesting data beyond the schema constraints of Avro conversion.
-However, the Parquet Avro Hadoop Parser was the original basis for supporting the Parquet format, and as such it is a bit more mature.
-
-#### Examples
-
-##### `parquet` parser, `parquet` parseSpec
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetInputFormat",
- "paths": "path/to/file.parquet"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "parquet",
- "parseSpec": {
- "format": "parquet",
- "flattenSpec": {
- "useFieldDiscovery": true,
- "fields": [
- {
- "type": "path",
- "name": "nestedDim",
- "expr": "$.nestedData.dim1"
- },
- {
- "type": "path",
- "name": "listDimFirstItem",
- "expr": "$.listDim[1]"
- }
- ]
- },
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [],
- "dimensionExclusions": [],
- "spatialDimensions": []
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
- }
-}
-```
-
-##### `parquet` parser, `timeAndDims` parseSpec
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetInputFormat",
- "paths": "path/to/file.parquet"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "parquet",
- "parseSpec": {
- "format": "timeAndDims",
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [
- "dim1",
- "dim2",
- "dim3",
- "listDim"
- ],
- "dimensionExclusions": [],
- "spatialDimensions": []
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
-}
-
-```
-
-### Parquet Avro Hadoop Parser
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-:::
-
-
-:::info
- Consider using the [Parquet Hadoop Parser](#parquet-hadoop-parser) over this parser to ingest
-Parquet files. See [Parquet Hadoop Parser vs Parquet Avro Hadoop Parser](#parquet-hadoop-parser-vs-parquet-avro-hadoop-parser)
-for the differences between those parsers.
-:::
-
-:::info
- You need to include both the [`druid-parquet-extensions`](../development/extensions-core/parquet.md)
-[`druid-avro-extensions`] as extensions to use the Parquet Avro Hadoop Parser.
-:::
-
-The Parquet Avro Hadoop Parser is for [Hadoop batch ingestion](./hadoop.md).
-This parser first converts the Parquet data into Avro records, and then parses them to ingest into Druid.
-The `inputFormat` of `inputSpec` in `ioConfig` must be set to `org.apache.druid.data.input.parquet.DruidParquetAvroInputFormat`.
-
-The Parquet Avro Hadoop Parser supports auto field discovery and flattening if provided with a
-[`flattenSpec`](#flattenspec) with the `avro` `parseSpec`. Parquet nested list and map
-[logical types](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md) _should_ operate correctly with
-JSON path expressions for all supported types. This parser sets a hadoop job property
-`parquet.avro.add-list-element-records` to `false` (which normally defaults to `true`), in order to 'unwrap' primitive
-list elements into multi-value dimensions.
-
-Note that the `int96` Parquet value type is not supported with this parser.
-
-|Field | Type | Description | Required|
-|----------|-------------|----------------------------------------------------------------------------------------|---------|
-| type | String | Set value to `parquet-avro`. | yes |
-| parseSpec | JSON Object | Specifies the timestamp and dimensions of the data, and optionally, a flatten spec. Should be `avro`. | yes |
-| binaryAsString | Boolean | Specifies if the bytes parquet column which is not logically marked as a string or enum type should be treated as a UTF-8 encoded string. | no(default = false) |
-
-When the time dimension is a [DateType column](https://github.com/apache/parquet-format/blob/master/LogicalTypes.md),
-a format should not be supplied. When the format is UTF8 (String), either `auto` or
-an explicitly defined [format](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat) is required.
-
-#### Example
-
-```json
-{
- "type": "index_hadoop",
- "spec": {
- "ioConfig": {
- "type": "hadoop",
- "inputSpec": {
- "type": "static",
- "inputFormat": "org.apache.druid.data.input.parquet.DruidParquetAvroInputFormat",
- "paths": "path/to/file.parquet"
- },
- ...
- },
- "dataSchema": {
- "dataSource": "example",
- "parser": {
- "type": "parquet-avro",
- "parseSpec": {
- "format": "avro",
- "flattenSpec": {
- "useFieldDiscovery": true,
- "fields": [
- {
- "type": "path",
- "name": "nestedDim",
- "expr": "$.nestedData.dim1"
- },
- {
- "type": "path",
- "name": "listDimFirstItem",
- "expr": "$.listDim[1]"
- }
- ]
- },
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [],
- "dimensionExclusions": [],
- "spatialDimensions": []
- }
- }
- },
- ...
- },
- "tuningConfig": <hadoop-tuning-config>
- }
- }
-}
-```
## ParseSpec
@@ -1642,7 +1067,7 @@ Also, you can skip some header rows by setting `skipHeaderRows` in your parseSpe
`skipHeaderRows` is first applied. For example, if you set `skipHeaderRows` to 2 and `hasHeaderRow` to true, Druid will
skip the first two lines and then extract column information from the third line.
-Note that `hasHeaderRow` and `skipHeaderRows` are effective only for non-Hadoop batch index tasks. Other types of index
+Note that `hasHeaderRow` and `skipHeaderRows` are effective only for batch index tasks. Other types of index
tasks will fail with an exception.
#### Other CSV Ingestion Tasks
@@ -1691,7 +1116,7 @@ Also, you can skip some header rows by setting `skipHeaderRows` in your parseSpe
`skipHeaderRows` is first applied. For example, if you set `skipHeaderRows` to 2 and `hasHeaderRow` to true, Druid will
skip the first two lines and then extract column information from the third line.
-Note that `hasHeaderRow` and `skipHeaderRows` are effective only for non-Hadoop batch index tasks. Other types of index
+Note that `hasHeaderRow` and `skipHeaderRows` are effective only for batch index tasks. Other types of index
tasks will fail with an exception.
#### Other TSV (Delimited) Ingestion Tasks
@@ -1749,25 +1174,3 @@ handle all formatting decisions on their own, without using the ParseSpec.
| format | String | `timeAndDims` | yes |
| timestampSpec | JSON Object | Specifies the column and format of the timestamp. | yes |
| dimensionsSpec | JSON Object | Specifies the dimensions of the data. | yes |
-
-### Orc ParseSpec
-
-Use this with the Hadoop ORC Parser to load ORC files.
-
-| Field | Type | Description | Required |
-|-------|------|-------------|----------|
-| format | String | `orc`| no |
-| timestampSpec | JSON Object | Specifies the column and format of the timestamp. | yes |
-| dimensionsSpec | JSON Object | Specifies the dimensions of the data. | yes |
-| flattenSpec | JSON Object | Specifies flattening configuration for nested JSON data. See [`flattenSpec`](#flattenspec) for more info. | no |
-
-### Parquet ParseSpec
-
-Use this with the Hadoop Parquet Parser to load Parquet files.
-
-| Field | Type | Description | Required |
-|-------|------|-------------|----------|
-| format | String | `parquet`| no |
-| timestampSpec | JSON Object | Specifies the column and format of the timestamp. | yes |
-| dimensionsSpec | JSON Object | Specifies the dimensions of the data. | yes |
-| flattenSpec | JSON Object | Specifies flattening configuration for nested JSON data. See [`flattenSpec`](#flattenspec) for more info. | no |
diff --git a/docs/ingestion/hadoop.md b/docs/ingestion/hadoop.md
index af416c87737..35c4eb373e2 100644
--- a/docs/ingestion/hadoop.md
+++ b/docs/ingestion/hadoop.md
@@ -23,558 +23,11 @@ sidebar_label: "Hadoop-based"
~ under the License.
-->
-:::caution[Deprecated]
+Support for Apache Hadoop-based ingestion was removed from Apache Druid 37.0.0. Please use
+[SQL-based ingestion](../multi-stage-query/index.md) or [native batch](../ingestion/native-batch.md) instead.
-Hadoop-based ingestion is deprecated and scheduled to be removed with Druid 37.0.0.
+The associated `materialized-view-selection` and `materialized-view-maintenance` contrib extensions were also removed
+as part of this since they only supported Hadoop based ingestion.
-We recommend one of Druid's other supported ingestion methods, such as [SQL-based ingestion](../multi-stage-query/index.md) or [MiddleManager-less ingestion using Kubernetes](../development/extensions-core/k8s-jobs.md)
-
-You must now explicitly opt-in to using the deprecated `index_hadoop` task type. To opt-in, set `druid.indexer.task.allowHadoopTaskExecution` to `true` in your `common.runtime.properties` file. For more information, see [#18239](https://github.com/apache/druid/pull/18239).
-
-:::
-
-
-
-Apache Hadoop-based batch ingestion in Apache Druid is supported via a Hadoop-ingestion task. These tasks can be posted to a running
-instance of a Druid [Overlord](../design/overlord.md). Please refer to our [Hadoop-based vs. native batch comparison table](index.md#batch) for
-comparisons between Hadoop-based, native batch (simple), and native batch (parallel) ingestion.
-
-To run a Hadoop-based ingestion task, write an ingestion spec as specified below. Then POST it to the
-[`/druid/indexer/v1/task`](../api-reference/tasks-api.md) endpoint on the Overlord, or use the
-`bin/post-index-task` script included with Druid.
-
-## Tutorial
-
-This page contains reference documentation for Hadoop-based ingestion.
-For a walk-through instead, check out the [Loading from Apache Hadoop](../tutorials/tutorial-batch-hadoop.md) tutorial.
-
-## Task syntax
-
-A sample task is shown below:
-
-```json
-{
- "type" : "index_hadoop",
- "spec" : {
- "dataSchema" : {
- "dataSource" : "wikipedia",
- "parser" : {
- "type" : "hadoopyString",
- "parseSpec" : {
- "format" : "json",
- "timestampSpec" : {
- "column" : "timestamp",
- "format" : "auto"
- },
- "dimensionsSpec" : {
- "dimensions": ["page","language","user","unpatrolled","newPage","robot","anonymous","namespace","continent","country","region","city"],
- "dimensionExclusions" : [],
- "spatialDimensions" : []
- }
- }
- },
- "metricsSpec" : [
- {
- "type" : "count",
- "name" : "count"
- },
- {
- "type" : "doubleSum",
- "name" : "added",
- "fieldName" : "added"
- },
- {
- "type" : "doubleSum",
- "name" : "deleted",
- "fieldName" : "deleted"
- },
- {
- "type" : "doubleSum",
- "name" : "delta",
- "fieldName" : "delta"
- }
- ],
- "granularitySpec" : {
- "type" : "uniform",
- "segmentGranularity" : "DAY",
- "queryGranularity" : "NONE",
- "intervals" : [ "2013-08-31/2013-09-01" ]
- }
- },
- "ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "static",
- "paths" : "/MyDirectory/example/wikipedia_data.json"
- }
- },
- "tuningConfig" : {
- "type": "hadoop"
- }
- },
- "hadoopDependencyCoordinates": <my_hadoop_version>
-}
-```
-
-|property|description|required?|
-|--------|-----------|---------|
-|type|The task type, this should always be "index_hadoop".|yes|
-|spec|A Hadoop Index Spec. See [Ingestion](../ingestion/index.md)|yes|
-|hadoopDependencyCoordinates|A JSON array of Hadoop dependency coordinates that Druid will use, this property will override the default Hadoop coordinates. Once specified, Druid will look for those Hadoop dependencies from the location specified by `druid.extensions.hadoopDependenciesDir`|no|
-|classpathPrefix|Classpath that will be prepended for the Peon process.|no|
-
-Also note that Druid automatically computes the classpath for Hadoop job containers that run in the Hadoop cluster. But in case of conflicts between Hadoop and Druid's dependencies, you can manually specify the classpath by setting `druid.extensions.hadoopContainerDruidClasspath` property. See the extensions config in [base druid configuration](../configuration/index.md#extensions).
-
-## `dataSchema`
-
-This field is required. See the [`dataSchema`](ingestion-spec.md#legacy-dataschema-spec) section of the main ingestion page for details on
-what it should contain.
-
-## `ioConfig`
-
-This field is required.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|type|String|This should always be 'hadoop'.|yes|
-|inputSpec|Object|A specification of where to pull the data in from. See below.|yes|
-|segmentOutputPath|String|The path to dump segments into.|Only used by the [Command-line Hadoop indexer](#cli). This field must be null otherwise.|
-|metadataUpdateSpec|Object|A specification of how to update the metadata for the druid cluster these segments belong to.|Only used by the [Command-line Hadoop indexer](#cli). This field must be null otherwise.|
-
-### `inputSpec`
-
-There are multiple types of inputSpecs:
-
-#### `static`
-
-A type of inputSpec where a static path to the data files is provided.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|inputFormat|String|Specifies the Hadoop InputFormat class to use. e.g. `org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat` |no|
-|paths|String|Comma-separated input paths to the raw data. Druid ingests data only from the configured paths. It does not search recursively for data in subdirectories. |yes|
-
-For example, using the static input paths:
-
-```
-"paths" : "hdfs://path/to/data/is/here/data.gz,hdfs://path/to/data/is/here/moredata.gz,hdfs://path/to/data/is/here/evenmoredata.gz"
-```
-
-You can also read from cloud storage such as Amazon S3 or Google Cloud Storage.
-To do so, you need to install the necessary library under Druid's classpath in _all Middle Manager or Indexer processes_.
-For S3, you can run the below command to install the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/).
-
-```bash
-java -classpath "${DRUID_HOME}lib/*" org.apache.druid.cli.Main tools pull-deps -h "org.apache.hadoop:hadoop-aws:${HADOOP_VERSION}";
-cp ${DRUID_HOME}/hadoop-dependencies/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar ${DRUID_HOME}/extensions/druid-hdfs-storage/
-```
-
-Once you install the Hadoop AWS module in all Middle Manager and Indexer processes, you can put
-your S3 paths in the inputSpec with the below job properties.
-For more configurations, see the [Hadoop AWS module](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/).
-
-```
-"paths" : "s3a://billy-bucket/the/data/is/here/data.gz,s3a://billy-bucket/the/data/is/here/moredata.gz,s3a://billy-bucket/the/data/is/here/evenmoredata.gz"
-```
-
-```json
-"jobProperties" : {
- "fs.s3a.impl" : "org.apache.hadoop.fs.s3a.S3AFileSystem",
- "fs.AbstractFileSystem.s3a.impl" : "org.apache.hadoop.fs.s3a.S3A",
- "fs.s3a.access.key" : "YOUR_ACCESS_KEY",
- "fs.s3a.secret.key" : "YOUR_SECRET_KEY"
-}
-```
-
-For Google Cloud Storage, you need to install [GCS connector jar](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md)
-under `${DRUID_HOME}/hadoop-dependencies` in _all Middle Manager or Indexer processes_.
-Once you install the GCS Connector jar in all Middle Manager and Indexer processes, you can put
-your Google Cloud Storage paths in the inputSpec with the below job properties.
-For more configurations, see the [instructions to configure Hadoop](https://github.com/GoogleCloudPlatform/bigdata-interop/blob/master/gcs/INSTALL.md#configure-hadoop),
-[GCS core default](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/v2.0.0/gcs/conf/gcs-core-default.xml)
-and [GCS core template](https://github.com/GoogleCloudDataproc/hadoop-connectors/blob/master/gcs/src/test/resources/core-site.xml).
-
-```
-"paths" : "gs://billy-bucket/the/data/is/here/data.gz,gs://billy-bucket/the/data/is/here/moredata.gz,gs://billy-bucket/the/data/is/here/evenmoredata.gz"
-```
-
-```json
-"jobProperties" : {
- "fs.gs.impl" : "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem",
- "fs.AbstractFileSystem.gs.impl" : "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS"
-}
-```
-
-#### `granularity`
-
-A type of inputSpec that expects data to be organized in directories according to datetime using the path format: `y=XXXX/m=XX/d=XX/H=XX/M=XX/S=XX` (where date is represented by lowercase and time is represented by uppercase).
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|dataGranularity|String|Specifies the granularity to expect the data at, e.g. hour means to expect directories `y=XXXX/m=XX/d=XX/H=XX`.|yes|
-|inputFormat|String|Specifies the Hadoop InputFormat class to use. e.g. `org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat` |no|
-|inputPath|String|Base path to append the datetime path to.|yes|
-|filePattern|String|Pattern that files should match to be included.|yes|
-|pathFormat|String|Joda datetime format for each directory. Default value is `"'y'=yyyy/'m'=MM/'d'=dd/'H'=HH"`, or see [Joda documentation](http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html)|no|
-
-For example, if the sample config were run with the interval 2012-06-01/2012-06-02, it would expect data at the paths:
-
-```
-s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=00
-s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=01
-...
-s3n://billy-bucket/the/data/is/here/y=2012/m=06/d=01/H=23
-```
-
-#### `dataSource`
-
-This is a type of `inputSpec` that reads data already stored inside Druid. This is used to allow "re-indexing" data and for "delta-ingestion" described later in `multi` type inputSpec.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|type|String.|This should always be 'dataSource'.|yes|
-|ingestionSpec|JSON object.|Specification of Druid segments to be loaded. See below.|yes|
-|maxSplitSize|Number|Enables combining multiple segments into single Hadoop InputSplit according to size of segments. With -1, druid calculates max split size based on user specified number of map task(mapred.map.tasks or mapreduce.job.maps). By default, one split is made for one segment. maxSplitSize is specified in bytes.|no|
-|useNewAggs|Boolean|If "false", then list of aggregators in "metricsSpec" of hadoop indexing task must be same as that used in original indexing task while ingesting raw data. Default value is "false". This field can be set to "true" when "inputSpec" type is "dataSource" and not "multi" to enable arbitrary aggregators while reindexing. See below for "multi" type support for delta-ingestion.|no|
-
-Here is what goes inside `ingestionSpec`:
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|dataSource|String|Druid dataSource name from which you are loading the data.|yes|
-|intervals|List|A list of strings representing ISO-8601 Intervals.|yes|
-|segments|List|List of segments from which to read data from, by default it is obtained automatically. You can obtain list of segments to put here by making a POST query to Coordinator at url /druid/coordinator/v1/metadata/datasources/segments?full with list of intervals specified in the request payload, e.g. ["2012-01-01T00:00:00.000/2012-01-03T00:00:00.000", "2012-01-05T00:00:00.000/2012-01-07T00:00:00.000"]. You may want to provide this list manually in order to ensure that segments r [...]
-|filter|JSON|See [Filters](../querying/filters.md)|no|
-|dimensions|Array of String|Name of dimension columns to load. By default, the list will be constructed from parseSpec. If parseSpec does not have an explicit list of dimensions then all the dimension columns present in stored data will be read.|no|
-|metrics|Array of String|Name of metric columns to load. By default, the list will be constructed from the "name" of all the configured aggregators.|no|
-|ignoreWhenNoSegments|boolean|Whether to ignore this ingestionSpec if no segments were found. Default behavior is to throw error when no segments were found.|no|
-
-For example
-
-```json
-"ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "dataSource",
- "ingestionSpec" : {
- "dataSource": "wikipedia",
- "intervals": ["2014-10-20T00:00:00Z/P2W"]
- }
- },
- ...
-}
-```
-
-#### `multi`
-
-This is a composing inputSpec to combine other inputSpecs. This inputSpec is used for delta ingestion. You can also use a `multi` inputSpec to combine data from multiple dataSources. However, each particular dataSource can only be specified one time.
-Note that, "useNewAggs" must be set to default value false to support delta-ingestion.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|children|Array of JSON objects|List of JSON objects containing other inputSpecs.|yes|
-
-For example:
-
-```json
-"ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "multi",
- "children": [
- {
- "type" : "dataSource",
- "ingestionSpec" : {
- "dataSource": "wikipedia",
- "intervals": ["2012-01-01T00:00:00.000/2012-01-03T00:00:00.000", "2012-01-05T00:00:00.000/2012-01-07T00:00:00.000"],
- "segments": [
- {
- "dataSource": "test1",
- "interval": "2012-01-01T00:00:00.000/2012-01-03T00:00:00.000",
- "version": "v2",
- "loadSpec": {
- "type": "local",
- "path": "/tmp/index1.zip"
- },
- "dimensions": "host",
- "metrics": "visited_sum,unique_hosts",
- "shardSpec": {
- "type": "none"
- },
- "binaryVersion": 9,
- "size": 2,
- "identifier": "test1_2000-01-01T00:00:00.000Z_3000-01-01T00:00:00.000Z_v2"
- }
- ]
- }
- },
- {
- "type" : "static",
- "paths": "/path/to/more/wikipedia/data/"
- }
- ]
- },
- ...
-}
-```
-
-It is STRONGLY RECOMMENDED to provide list of segments in `dataSource` inputSpec explicitly so that your delta ingestion task is idempotent. You can obtain that list of segments by making following call to the Coordinator.
-POST `/druid/coordinator/v1/metadata/datasources/{dataSourceName}/segments?full`
-Request Body: [interval1, interval2,...] for example ["2012-01-01T00:00:00.000/2012-01-03T00:00:00.000", "2012-01-05T00:00:00.000/2012-01-07T00:00:00.000"]
-
-## `tuningConfig`
-
-The tuningConfig is optional and default parameters will be used if no tuningConfig is specified.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|workingPath|String|The working path to use for intermediate results (results between Hadoop jobs).|Only used by the [Command-line Hadoop indexer](#cli). The default is '/tmp/druid-indexing'. This field must be null otherwise.|
-|version|String|The version of created segments. Ignored for HadoopIndexTask unless useExplicitVersion is set to true|no (default == datetime that indexing starts at)|
-|partitionsSpec|Object|A specification of how to partition each time bucket into segments. Absence of this property means no partitioning will occur. See [`partitionsSpec`](#partitionsspec) below.|no (default == 'hashed')|
-|maxRowsInMemory|Integer|The number of rows to aggregate before persisting. Note that this is the number of post-aggregation rows which may not be equal to the number of input events due to roll-up. This is used to manage the required JVM heap size. Normally user does not need to set this, but depending on the nature of data, if rows are short in terms of bytes, user may not want to store a million rows in memory and this value should be set.|no (default == 1000000)|
-|maxBytesInMemory|Long|The number of bytes to aggregate in heap memory before persisting. Normally this is computed internally and user does not need to set it. This is based on a rough estimate of memory usage and not actual usage. The maximum heap memory usage for indexing is maxBytesInMemory * (2 + maxPendingPersists). Note that `maxBytesInMemory` also includes heap usage of artifacts created from intermediary persists. This means that after every persist, the amount of `maxBytesInMem [...]
-|leaveIntermediate|Boolean|Leave behind intermediate files (for debugging) in the workingPath when a job completes, whether it passes or fails.|no (default == false)|
-|cleanupOnFailure|Boolean|Clean up intermediate files when a job fails (unless leaveIntermediate is on).|no (default == true)|
-|overwriteFiles|Boolean|Override existing files found during indexing.|no (default == false)|
-|ignoreInvalidRows|Boolean|DEPRECATED. Ignore rows found to have problems. If false, any exception encountered during parsing will be thrown and will halt ingestion; if true, unparseable rows and fields will be skipped. If `maxParseExceptions` is defined, this property is ignored.|no (default == false)|
-|combineText|Boolean|Use CombineTextInputFormat to combine multiple files into a file split. This can speed up Hadoop jobs when processing a large number of small files.|no (default == false)|
-|useCombiner|Boolean|Use Hadoop combiner to merge rows at mapper if possible.|no (default == false)|
-|jobProperties|Object|A map of properties to add to the Hadoop job configuration, see below for details.|no (default == null)|
-|indexSpec|Object|Tune how data is indexed. See [`indexSpec`](ingestion-spec.md#indexspec) on the main ingestion page for more information.|no|
-|indexSpecForIntermediatePersists|Object|defines segment storage format options to be used at indexing time for intermediate persisted temporary segments. this can be used to disable dimension/metric compression on intermediate segments to reduce memory required for final merging. however, disabling compression on intermediate segments might increase page cache use while they are used before getting merged into final segment published, see [`indexSpec`](ingestion-spec.md#indexspec) for p [...]
-|numBackgroundPersistThreads|Integer|The number of new background threads to use for incremental persists. Using this feature causes a notable increase in memory pressure and CPU usage but will make the job finish more quickly. If changing from the default of 0 (use current thread for persists), we recommend setting it to 1.|no (default == 0)|
-|forceExtendableShardSpecs|Boolean|Forces use of extendable shardSpecs. Hash-based partitioning always uses an extendable shardSpec. For single-dimension partitioning, this option should be set to true to use an extendable shardSpec. For partitioning, please check [Partitioning specification](#partitionsspec). This option can be useful when you need to append more data to existing dataSource.|no (default = false)|
-|useExplicitVersion|Boolean|Forces HadoopIndexTask to use version.|no (default = false)|
-|logParseExceptions|Boolean|If true, log an error message when a parsing exception occurs, containing information about the row where the error occurred.|no(default = false)|
-|maxParseExceptions|Integer|The maximum number of parse exceptions that can occur before the task halts ingestion and fails. Overrides `ignoreInvalidRows` if `maxParseExceptions` is defined.|no(default = unlimited)|
-|useYarnRMJobStatusFallback|Boolean|If the Hadoop jobs created by the indexing task are unable to retrieve their completion status from the JobHistory server, and this parameter is true, the indexing task will try to fetch the application status from `http://<yarn-rm-address>/ws/v1/cluster/apps/<application-id>`, where `<yarn-rm-address>` is the value of `yarn.resourcemanager.webapp.address` in your Hadoop configuration. This flag is intended as a fallback for cases where an indexing tas [...]
-|awaitSegmentAvailabilityTimeoutMillis|Long|Milliseconds to wait for the newly indexed segments to become available for query after ingestion completes. If `<= 0`, no wait will occur. If `> 0`, the task will wait for the Coordinator to indicate that the new segments are available for querying. If the timeout expires, the task will exit as successful, but the segments were not confirmed to have become available for query.|no (default = 0)|
-
-### `jobProperties`
-
-```json
- "tuningConfig" : {
- "type": "hadoop",
- "jobProperties": {
- "<hadoop-property-a>": "<value-a>",
- "<hadoop-property-b>": "<value-b>"
- }
- }
-```
-
-Hadoop's [MapReduce documentation](https://hadoop.apache.org/docs/stable/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml) lists the possible configuration parameters.
-
-With some Hadoop distributions, it may be necessary to set `mapreduce.job.classpath` or `mapreduce.job.user.classpath.first`
-to avoid class loading issues. See the [working with different Hadoop versions documentation](../operations/other-hadoop.md)
-for more details.
-
-## `partitionsSpec`
-
-Segments are always partitioned based on timestamp (according to the granularitySpec) and may be further partitioned in
-some other way depending on partition type. Druid supports two types of partitioning strategies: `hashed` (based on the
-hash of all dimensions in each row), and `single_dim` (based on ranges of a single dimension).
-
-Hashed partitioning is recommended in most cases, as it will improve indexing performance and create more uniformly
-sized data segments relative to single-dimension partitioning.
-
-### Hash-based partitioning
-
-```json
- "partitionsSpec": {
- "type": "hashed",
- "targetRowsPerSegment": 5000000
- }
-```
-
-Hashed partitioning works by first selecting a number of segments, and then partitioning rows across those segments
-according to the hash of all dimensions in each row. The number of segments is determined automatically based on the
-cardinality of the input set and a target partition size.
-
-The configuration options are:
-
-|Field|Description|Required|
-|--------|-----------|---------|
-|type|Type of partitionSpec to be used.|"hashed"|
-|targetRowsPerSegment|Target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB. Defaults to 5000000 if `numShards` is not set.|either this or `numShards`|
-|targetPartitionSize|Deprecated. Renamed to `targetRowsPerSegment`. Target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|either this or `numShards`|
-|maxRowsPerSegment|Deprecated. Renamed to `targetRowsPerSegment`. Target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|either this or `numShards`|
-|numShards|Specify the number of partitions directly, instead of a target partition size. Ingestion will run faster, since it can skip the step necessary to select a number of partitions automatically.|either this or `targetRowsPerSegment`|
-|partitionDimensions|The dimensions to partition on. Leave blank to select all dimensions. Only used with `numShards`, will be ignored when `targetRowsPerSegment` is set.|no|
-|partitionFunction|A function to compute hash of partition dimensions. See [Hash partition function](#hash-partition-function)|`murmur3_32_abs`|no|
-
-##### Hash partition function
-
-In hash partitioning, the partition function is used to compute hash of partition dimensions. The partition dimension
-values are first serialized into a byte array as a whole, and then the partition function is applied to compute hash of
-the byte array.
-Druid currently supports only one partition function.
-
-|name|description|
-|----|-----------|
-|`murmur3_32_abs`|Applies an absolute value function to the result of [`murmur3_32`](https://guava.dev/releases/16.0/api/docs/com/google/common/hash/Hashing.html#murmur3_32()).|
-
-### Single-dimension range partitioning
-
-```json
- "partitionsSpec": {
- "type": "single_dim",
- "targetRowsPerSegment": 5000000
- }
-```
-
-Single-dimension range partitioning works by first selecting a dimension to partition on, and then separating that dimension
-into contiguous ranges. Each segment will contain all rows with values of that dimension in that range. For example,
-your segments may be partitioned on the dimension "host" using the ranges "a.example.com" to "f.example.com" and
-"f.example.com" to "z.example.com". By default, the dimension to use is determined automatically, although you can
-override it with a specific dimension.
-
-The configuration options are:
-
-|Field|Description|Required|
-|--------|-----------|---------|
-|type|Type of partitionSpec to be used.|"single_dim"|
-|targetRowsPerSegment|Target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|yes|
-|targetPartitionSize|Deprecated. Renamed to `targetRowsPerSegment`. Target number of rows to include in a partition, should be a number that targets segments of 500MB\~1GB.|no|
-|maxRowsPerSegment|Maximum number of rows to include in a partition. Defaults to 50% larger than the `targetRowsPerSegment`.|no|
-|maxPartitionSize|Deprecated. Use `maxRowsPerSegment` instead. Maximum number of rows to include in a partition. Defaults to 50% larger than the `targetPartitionSize`.|no|
-|partitionDimension|The dimension to partition on. Leave blank to select a dimension automatically.|no|
-|assumeGrouped|Assume that input data has already been grouped on time and dimensions. Ingestion will run faster, but may choose sub-optimal partitions if this assumption is violated.|no|
-
-## Remote Hadoop clusters
-
-If you have a remote Hadoop cluster, make sure to include the folder holding your configuration `*.xml` files in your Druid `_common` configuration folder.
-
-If you are having dependency problems with your version of Hadoop and the version compiled with Druid, please see [these docs](../operations/other-hadoop.md).
-
-## Elastic MapReduce
-
-If your cluster is running on Amazon Web Services, you can use Elastic MapReduce (EMR) to index data
-from S3. To do this:
-
-- Create a persistent, [long-running cluster](http://docs.aws.amazon.com/ElasticMapReduce/latest/ManagementGuide/emr-plan-longrunning-transient).
-- When creating your cluster, enter the following configuration. If you're using the wizard, this
-should be in advanced mode under "Edit software settings":
-
-```
-classification=yarn-site,properties=[mapreduce.reduce.memory.mb=6144,mapreduce.reduce.java.opts=-server -Xms2g -Xmx2g -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps,mapreduce.map.java.opts=758,mapreduce.map.java.opts=-server -Xms512m -Xmx512m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps,mapreduce.task.timeout=1800000]
-```
-
-- Follow the instructions under
-[Configure for connecting to Hadoop](../tutorials/cluster.md#configure-for-connecting-to-hadoop-optional) using the XML files from `/etc/hadoop/conf`
-on your EMR master.
-
-## Kerberized Hadoop clusters
-
-By default druid can use the existing TGT kerberos ticket available in local kerberos key cache.
-Although TGT ticket has a limited life cycle,
-therefore you need to call `kinit` command periodically to ensure validity of TGT ticket.
-To avoid this extra external cron job script calling `kinit` periodically,
- you can provide the principal name and keytab location and druid will do the authentication transparently at startup and job launching time.
-
-|Property|Possible Values|Description|Default|
-|--------|---------------|-----------|-------|
-|`druid.hadoop.security.kerberos.principal`|`druid@EXAMPLE.COM`| Principal user name |empty|
-|`druid.hadoop.security.kerberos.keytab`|`/etc/security/keytabs/druid.headlessUser.keytab`|Path to keytab file|empty|
-
-### Loading from S3 with EMR
-
-- In the `jobProperties` field in the `tuningConfig` section of your Hadoop indexing task, add:
-
-```
-"jobProperties" : {
- "fs.s3.awsAccessKeyId" : "YOUR_ACCESS_KEY",
- "fs.s3.awsSecretAccessKey" : "YOUR_SECRET_KEY",
- "fs.s3.impl" : "org.apache.hadoop.fs.s3native.NativeS3FileSystem",
- "fs.s3n.awsAccessKeyId" : "YOUR_ACCESS_KEY",
- "fs.s3n.awsSecretAccessKey" : "YOUR_SECRET_KEY",
- "fs.s3n.impl" : "org.apache.hadoop.fs.s3native.NativeS3FileSystem",
- "io.compression.codecs" : "org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.SnappyCodec"
-}
-```
-
-Note that this method uses Hadoop's built-in S3 filesystem rather than Amazon's EMRFS, and is not compatible
-with Amazon-specific features such as S3 encryption and consistent views. If you need to use these
-features, you will need to make the Amazon EMR Hadoop JARs available to Druid through one of the
-mechanisms described in the [Using other Hadoop distributions](#using-other-hadoop-distributions) section.
-
-## Using other Hadoop distributions
-
-Druid works out of the box with many Hadoop distributions.
-
-If you are having dependency conflicts between Druid and your version of Hadoop, you can try
-searching for a solution in the [Druid user groups](https://groups.google.com/forum/#!forum/druid-user), or reading the
-Druid [Different Hadoop Versions](../operations/other-hadoop.md) documentation.
-
-<a name="cli"></a>
-
-## Command line (non-task) version
-
-To run:
-
-```
-java -Xmx256m -Duser.timezone=UTC -Dfile.encoding=UTF-8 -classpath lib/*:<hadoop_config_dir> org.apache.druid.cli.Main index hadoop <spec_file>
-```
-
-### Options
-
-- "--coordinate" - provide a version of Apache Hadoop to use. This property will override the default Hadoop coordinates. Once specified, Apache Druid will look for those Hadoop dependencies from the location specified by `druid.extensions.hadoopDependenciesDir`.
-- "--no-default-hadoop" - don't pull down the default hadoop version
-
-### Spec file
-
-The spec file needs to contain a JSON object where the contents are the same as the "spec" field in the Hadoop index task. See [Hadoop Batch Ingestion](../ingestion/hadoop.md) for details on the spec format.
-
-In addition, a `metadataUpdateSpec` and `segmentOutputPath` field needs to be added to the ioConfig:
-
-```
- "ioConfig" : {
- ...
- "metadataUpdateSpec" : {
- "type":"mysql",
- "connectURI" : "jdbc:mysql://localhost:3306/druid",
- "password" : "diurd",
- "segmentTable" : "druid_segments",
- "user" : "druid"
- },
- "segmentOutputPath" : "/MyDirectory/data/index/output"
- },
-```
-
-and a `workingPath` field needs to be added to the tuningConfig:
-
-```
- "tuningConfig" : {
- ...
- "workingPath": "/tmp",
- ...
- }
-```
-
-#### Metadata Update Job Spec
-
-This is a specification of the properties that tell the job how to update metadata such that the Druid cluster will see the output segments and load them.
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|type|String|"metadata" is the only value available.|yes|
-|connectURI|String|A valid JDBC url to metadata storage.|yes|
-|user|String|Username for db.|yes|
-|password|String|password for db.|yes|
-|segmentTable|String|Table to use in DB.|yes|
-
-These properties should parrot what you have configured for your [Coordinator](../design/coordinator.md).
-
-#### segmentOutputPath Config
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|segmentOutputPath|String|the path to dump segments into.|yes|
-
-#### workingPath Config
-
-|Field|Type|Description|Required|
-|-----|----|-----------|--------|
-|workingPath|String|the working path to use for intermediate results (results between Hadoop jobs).|no (default == '/tmp/druid-indexing')|
-
-Please note that the command line Hadoop indexer doesn't have the locking capabilities of the indexing service, so if you choose to use it,
-you have to take caution to not override segments created by real-time processing (if you that a real-time pipeline set up).
+Note that Druid still supports using [`druid-hdfs-storage`](../development/extensions-core/hdfs.md) as deep storage and other Hadoop ecosystem extensions and
+functionality that was not specific to Hadoop-based ingestion.
diff --git a/docs/ingestion/index.md b/docs/ingestion/index.md
index de90051fca0..4d63fd64020 100644
--- a/docs/ingestion/index.md
+++ b/docs/ingestion/index.md
@@ -62,19 +62,19 @@ supervisor.
### Batch
-There are three available options for batch ingestion. Batch ingestion jobs are associated with a controller task that
+There are two available options for batch ingestion. Batch ingestion jobs are associated with a controller task that
runs for the duration of the job.
-| **Method** | [Native batch](./native-batch.md) | [SQL](../multi-stage-query/index.md) | [Hadoop-based (deprecated)](hadoop.md) |
-|---|-----|--------------|------------|
-| **Controller task type** | `index_parallel` | `query_controller` | `index_hadoop` |
-| **How you submit it** | Send an `index_parallel` spec to the [Tasks API](../api-reference/tasks-api.md). | Send an [INSERT](../multi-stage-query/concepts.md#load-data-with-insert) or [REPLACE](../multi-stage-query/concepts.md#overwrite-data-with-replace) statement to the [SQL task API](../api-reference/sql-ingestion-api.md#submit-a-query). | Send an `index_hadoop` spec to the [Tasks API](../api-reference/tasks-api.md). |
-| **Parallelism** | Using subtasks, if [`maxNumConcurrentSubTasks`](native-batch.md#tuningconfig) is greater than 1. | Using `query_worker` subtasks. | Using YARN. |
-| **Fault tolerance** | Workers automatically relaunched upon failure. Controller task failure leads to job failure. | Controller or worker task failure leads to job failure. | YARN containers automatically relaunched upon failure. Controller task failure leads to job failure. |
-| **Can append?** | Yes. | Yes (INSERT). | No. |
-| **Can overwrite?** | Yes. | Yes (REPLACE). | Yes. |
-| **External dependencies** | None. | None. | Hadoop cluster. |
-| **Input sources** | Any [`inputSource`](./input-sources.md). | Any [`inputSource`](./input-sources.md) (using [EXTERN](../multi-stage-query/concepts.md#write-to-an-external-destination-with-extern)) or Druid datasource (using FROM). | Any Hadoop FileSystem or Druid datasource. |
-| **Input formats** | Any [`inputFormat`](./data-formats.md#input-format). | Any [`inputFormat`](./data-formats.md#input-format). | Any Hadoop InputFormat. |
-| **Secondary partitioning options** | Dynamic, hash-based, and range-based partitioning methods are available. See [partitionsSpec](./native-batch.md#partitionsspec) for details.| Range partitioning ([CLUSTERED BY](../multi-stage-query/concepts.md#clustering)). | Hash-based or range-based partitioning via [`partitionsSpec`](hadoop.md#partitionsspec). |
-| **[Rollup modes](./rollup.md#perfect-rollup-vs-best-effort-rollup)** | Perfect if `forceGuaranteedRollup` = true in the [`tuningConfig`](native-batch.md#tuningconfig). | Always perfect. | Always perfect. |
+| **Method** | [Native batch](./native-batch.md) | [SQL](../multi-stage-query/index.md) |
+|---|-----|--------------|
+| **Controller task type** | `index_parallel` | `query_controller` |
+| **How you submit it** | Send an `index_parallel` spec to the [Tasks API](../api-reference/tasks-api.md). | Send an [INSERT](../multi-stage-query/concepts.md#load-data-with-insert) or [REPLACE](../multi-stage-query/concepts.md#overwrite-data-with-replace) statement to the [SQL task API](../api-reference/sql-ingestion-api.md#submit-a-query). |
+| **Parallelism** | Using subtasks, if [`maxNumConcurrentSubTasks`](native-batch.md#tuningconfig) is greater than 1. | Using `query_worker` subtasks. |
+| **Fault tolerance** | Workers automatically relaunched upon failure. Controller task failure leads to job failure. | Controller or worker task failure leads to job failure. |
+| **Can append?** | Yes. | Yes (INSERT). |
+| **Can overwrite?** | Yes. | Yes (REPLACE). |
+| **External dependencies** | None. | None. |
+| **Input sources** | Any [`inputSource`](./input-sources.md). | Any [`inputSource`](./input-sources.md) (using [EXTERN](../multi-stage-query/concepts.md#write-to-an-external-destination-with-extern)) or Druid datasource (using FROM). |
+| **Input formats** | Any [`inputFormat`](./data-formats.md#input-format). | Any [`inputFormat`](./data-formats.md#input-format). |
+| **Secondary partitioning options** | Dynamic, hash-based, and range-based partitioning methods are available. See [partitionsSpec](./native-batch.md#partitionsspec) for details.| Range partitioning ([CLUSTERED BY](../multi-stage-query/concepts.md#clustering)). |
+| **[Rollup modes](./rollup.md#perfect-rollup-vs-best-effort-rollup)** | Perfect if `forceGuaranteedRollup` = true in the [`tuningConfig`](native-batch.md#tuningconfig). | Always perfect. |
diff --git a/docs/ingestion/ingestion-spec.md b/docs/ingestion/ingestion-spec.md
index 4d607ada754..fc8b96f19d8 100644
--- a/docs/ingestion/ingestion-spec.md
+++ b/docs/ingestion/ingestion-spec.md
@@ -24,7 +24,7 @@ description: Reference for the configuration options in the ingestion spec.
~ under the License.
-->
-All ingestion methods use ingestion tasks to load data into Druid. Streaming ingestion uses ongoing supervisors that run and supervise a set of tasks over time. Native batch and Hadoop-based ingestion use a one-time [task](tasks.md). Other than with SQL-based ingestion, use an _ingestion spec_ to configure your ingestion.
+All ingestion methods use ingestion tasks to load data into Druid. Streaming ingestion uses ongoing supervisors that run and supervise a set of tasks over time. Native batch uses a one-time [task](tasks.md). Other than with SQL-based ingestion, use an _ingestion spec_ to configure your ingestion.
Ingestion specs consists of three main components:
@@ -102,10 +102,6 @@ available in Druid's [web console](../operations/web-console.md). Druid's visual
## `dataSchema`
-:::info
- The `dataSchema` spec has been changed in 0.17.0. The new spec is supported by all ingestion methods
-except for _Hadoop_ ingestion. See the [Legacy `dataSchema` spec](#legacy-dataschema-spec) for the old spec.
-:::
The `dataSchema` is a holder for the following components:
@@ -442,11 +438,6 @@ Projections you define become a dimension for your datasource. To remove a proje
### Legacy `dataSchema` spec
-:::info
- The `dataSchema` spec has been changed in 0.17.0. The new spec is supported by all ingestion methods
-except for _Hadoop_ ingestion. See [`dataSchema`](#dataschema) for the new spec.
-:::
-
The legacy `dataSchema` spec has below two more components in addition to the ones listed in the [`dataSchema`](#dataschema) section above.
- [input row parser](#parser-deprecated), [flattening of nested data](#flattenspec) (if needed)
@@ -501,8 +492,7 @@ See [Flatten spec](./data-formats.md#flattenspec) for more details.
The `ioConfig` influences how data is read from a source system, such as Apache Kafka, Amazon S3, a mounted
filesystem, or any other supported source system. The `inputFormat` property applies to all
-[ingestion method](./index.md#ingestion-methods) except for Hadoop ingestion. The Hadoop ingestion still
-uses the [`parser`](#parser-deprecated) in the legacy `dataSchema`.
+[ingestion method](./index.md#ingestion-methods).
The rest of `ioConfig` is specific to each individual ingestion method.
An example `ioConfig` to read JSON data is:
@@ -527,13 +517,13 @@ The following table lists the common tuning properties shared among ingestion me
|Field|Description|Default|
|-----|-----------|-------|
-|type|Each ingestion method has its own tuning type code. You must specify the type code that matches your ingestion method. Common options are `index`, `hadoop`, `kafka`, and `kinesis`.||
+|type|Each ingestion method has its own tuning type code. You must specify the type code that matches your ingestion method. Common options are `index`, `kafka`, and `kinesis`.||
|maxRowsInMemory|The maximum number of records to store in memory before persisting to disk. Note that this is the number of rowsย post-rollup, and so it may not be equal to the number of input records. Ingested records will be persisted to disk when either `maxRowsInMemory` or `maxBytesInMemory` are reached (whichever happens first).|`1000000`|
|maxBytesInMemory|The maximum aggregate size of records, in bytes, to store in the JVM heap before persisting. This is based on a rough estimate of memory usage. Ingested records will be persisted to disk when either `maxRowsInMemory` or `maxBytesInMemory` are reached (whichever happens first). `maxBytesInMemory` also includes heap usage of artifacts created from intermediary persists. This means that after every persist, the amount of `maxBytesInMemory` until the next persist will decre [...]
|skipBytesInMemoryOverheadCheck|The calculation of maxBytesInMemory takes into account overhead objects created during ingestion and each intermediate persist. Setting this to true can exclude the bytes of these overhead objects from maxBytesInMemory check.|false|
|indexSpec|Defines segment storage format options to use at indexing time.|See [`indexSpec`](#indexspec) for more information.|
|indexSpecForIntermediatePersists|Defines segment storage format options to use at indexing time for intermediate persisted temporary segments.|See [`indexSpec`](#indexspec) for more information.|
-|Other properties|Each ingestion method has its own list of additional tuning properties. See the documentation for each method for a full list: [Kafka indexing service](../ingestion/kafka-ingestion.md#tuning-configuration), [Kinesis indexing service](../ingestion/kinesis-ingestion.md#tuning-configuration), [Native batch](native-batch.md#tuningconfig), and [Hadoop-based](hadoop.md#tuningconfig).||
+|Other properties|Each ingestion method has its own list of additional tuning properties. See the documentation for each method for a full list: [Kafka indexing service](../ingestion/kafka-ingestion.md#tuning-configuration), [Kinesis indexing service](../ingestion/kinesis-ingestion.md#tuning-configuration), and [Native batch](native-batch.md#tuningconfig).||
The following example shows a `tuningConfig` object that sets all of the shared common properties to their defaults:
diff --git a/docs/ingestion/native-batch.md b/docs/ingestion/native-batch.md
index 0eb9ff3cc8d..50eaf43366d 100644
--- a/docs/ingestion/native-batch.md
+++ b/docs/ingestion/native-batch.md
@@ -24,7 +24,7 @@ sidebar_label: JSON-based batch
-->
:::info
- This page describes JSON-based batch ingestion using [ingestion specs](ingestion-spec.md). For SQL-based batch ingestion using the [`druid-multi-stage-query`](../multi-stage-query/index.md) extension, see [SQL-based ingestion](../multi-stage-query/index.md). Refer to the [ingestion methods](../ingestion/index.md#batch) table to determine which ingestion method is right for you.
+ This page describes JSON-based batch ingestion using [ingestion specs](ingestion-spec.md). For SQL-based batch ingestion using the [`druid-multi-stage-query`](../multi-stage-query/index.md) engine, see [SQL-based ingestion](../multi-stage-query/index.md). Refer to the [ingestion methods](../ingestion/index.md#batch) table to determine which ingestion method is right for you.
:::
Apache Druid supports the following types of JSON-based batch indexing tasks:
@@ -50,7 +50,7 @@ To run either kind of JSON-based batch indexing task, you can:
## Parallel task indexing
-The parallel task type `index_parallel` is a task for multi-threaded batch indexing. Parallel task indexing only relies on Druid resources. It doesn't depend on other external systems like Hadoop.
+The parallel task type `index_parallel` is a task for multi-threaded batch indexing.
The `index_parallel` task is a supervisor task that orchestrates
the whole indexing process. The supervisor task splits the input data and creates worker tasks to process the individual portions of data.
diff --git a/docs/ingestion/partitioning.md b/docs/ingestion/partitioning.md
index 4d4b4b4dac7..e0ffe079c9f 100644
--- a/docs/ingestion/partitioning.md
+++ b/docs/ingestion/partitioning.md
@@ -50,7 +50,7 @@ The following table describes how to configure time chunk partitioning.
|------|------------|
|[SQL](../multi-stage-query/index.md)|[`PARTITIONED BY`](../multi-stage-query/concepts.md#partitioning-by-time)|
|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|`segmentGranularity` inside the [`granularitySpec`](ingestion-spec.md#granularityspec)|
-|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|`segmentGranularity` inside the [`granularitySpec`](ingestion-spec.md#granularityspec)|
+|[Native batch](native-batch.md)|`segmentGranularity` inside the [`granularitySpec`](ingestion-spec.md#granularityspec)|
## Secondary partitioning
@@ -66,7 +66,7 @@ The following table describes how to configure secondary partitioning.
|------|------------|
|[SQL](../multi-stage-query/index.md)|[`CLUSTERED BY`](../multi-stage-query/concepts.md#clustering)|
|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|Upstream partitioning defines how Druid partitions the datasource. You can also alter clustering using [`REPLACE`](../multi-stage-query/concepts.md#overwrite-data-with-replace) (with `CLUSTERED BY`) or [compaction](../data-management/compaction.md) after initial ingestion.|
-|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|[`partitionsSpec`](native-batch.md#partitionsspec) inside the `tuningConfig`|
+|[Native batch](native-batch.md)|[`partitionsSpec`](native-batch.md#partitionsspec) inside the `tuningConfig`|
## Sorting
@@ -82,7 +82,7 @@ The following table describes how to configure sorting.
|------|------------|
|[SQL](../multi-stage-query/index.md)|Uses order of fields in [`CLUSTERED BY`](../multi-stage-query/concepts.md#clustering) or [`segmentSortOrder`](../multi-stage-query/reference.md#context-parameters) in the query context|
|[Kafka](../ingestion/kafka-ingestion.md) or [Kinesis](../ingestion/kinesis-ingestion.md)|Uses order of fields in [`dimensionsSpec`](ingestion-spec.md#dimensionsspec)|
-|[Native batch](native-batch.md) or [Hadoop](hadoop.md)|Uses order of fields in [`dimensionsSpec`](ingestion-spec.md#dimensionsspec)|
+|[Native batch](native-batch.md)|Uses order of fields in [`dimensionsSpec`](ingestion-spec.md#dimensionsspec)|
:::info
Druid implicitly sorts rows within a segment by `__time` first before any `dimensions` or `CLUSTERED BY` fields, unless
diff --git a/docs/ingestion/rollup.md b/docs/ingestion/rollup.md
index 52ffc263336..db840270c45 100644
--- a/docs/ingestion/rollup.md
+++ b/docs/ingestion/rollup.md
@@ -86,7 +86,6 @@ The following table shows how each method handles rollup:
|------|------------|
|[Native batch](native-batch.md)|`index_parallel` and `index` type may be either perfect or best-effort, based on configuration.|
|[SQL-based batch](../multi-stage-query/index.md)|Always perfect.|
-|[Hadoop](hadoop.md)|Always perfect.|
|[Kafka indexing service](../ingestion/kafka-ingestion.md)|Always best-effort.|
|[Kinesis indexing service](../ingestion/kinesis-ingestion.md)|Always best-effort.|
diff --git a/docs/ingestion/tasks.md b/docs/ingestion/tasks.md
index 94381b48743..2e6cb771bc8 100644
--- a/docs/ingestion/tasks.md
+++ b/docs/ingestion/tasks.md
@@ -46,7 +46,7 @@ the Overlord APIs.
A report containing information about the number of rows ingested, and any parse exceptions that occurred is available for both completed tasks and running tasks.
-The reporting feature is supported by [native batch tasks](native-batch.md), the Hadoop batch task, and Kafka and Kinesis ingestion tasks.
+The reporting feature is supported by [native batch tasks](native-batch.md) and Kafka and Kinesis ingestion tasks.
### Completion report
@@ -244,7 +244,7 @@ The `ingestionState` shows what step of ingestion the task reached. Possible sta
Only batch tasks have the DETERMINE_PARTITIONS phase. Realtime tasks such as those created by the Kafka Indexing Service do not have a DETERMINE_PARTITIONS phase.
-`unparseableEvents` contains lists of exception messages that were caused by unparseable inputs. This can help with identifying problematic input rows. There will be one list each for the DETERMINE_PARTITIONS and BUILD_SEGMENTS phases. Note that the Hadoop batch task does not support saving of unparseable events.
+`unparseableEvents` contains lists of exception messages that were caused by unparseable inputs. This can help with identifying problematic input rows. There will be one list each for the DETERMINE_PARTITIONS and BUILD_SEGMENTS phases.
the `rowStats` map contains information about row counts. There is one entry for each ingestion phase. The definitions of the different row counts are shown below:
- `processed`: Number of rows successfully ingested without parsing errors
@@ -259,7 +259,7 @@ The `errorMsg` field shows a message describing the error that caused a task to
### Row stats
-The [native batch task](native-batch.md), the Hadoop batch task, and Kafka and Kinesis ingestion tasks support retrieval of row stats while the task is running.
+The [native batch task](native-batch.md) and Kafka and Kinesis ingestion tasks support retrieval of row stats while the task is running.
The live report can be accessed with a GET to the following URL on a Peon running a task:
@@ -388,7 +388,6 @@ Once `forceTimeChunkLock` is unset, the task will choose a proper lock type to u
Please note that segment lock is not always available. The most common use case where time chunk lock is enforced is
when an overwriting task changes the segment granularity.
Also, the segment locking is supported by only native indexing tasks and Kafka/Kinesis indexing tasks.
-Hadoop indexing tasks don't support it.
`forceTimeChunkLock` in the task context is only applied to individual tasks.
If you want to unset it for all tasks, you would want to set `druid.indexer.tasklock.forceTimeChunkLock` to false in the [overlord configuration](../configuration/index.md#overlord-operations).
@@ -421,7 +420,7 @@ Each task type has a different default lock priority. The below table shows the
|task type|default priority|
|---------|----------------|
|Realtime index task|75|
-|Batch index tasks, including [native batch](native-batch.md), [SQL](../multi-stage-query/index.md), and [Hadoop-based](hadoop.md)|50|
+|Batch index tasks, including [native batch](native-batch.md) and [SQL](../multi-stage-query/index.md)|50|
|Merge/Append/Compaction task|25|
|Other tasks|0|
@@ -524,10 +523,6 @@ The actual amount of memory assigned to any given task is computed by determinin
See [Native batch ingestion (parallel task)](native-batch.md).
-### `index_hadoop`
-
-See [Hadoop-based ingestion](hadoop.md).
-
### `index_kafka`
Submitted automatically, on your behalf, by a
diff --git a/docs/operations/basic-cluster-tuning.md b/docs/operations/basic-cluster-tuning.md
index 51f8cb0f037..7060b8c7bee 100644
--- a/docs/operations/basic-cluster-tuning.md
+++ b/docs/operations/basic-cluster-tuning.md
@@ -261,10 +261,6 @@ If you use the [Kafka Indexing Service](../ingestion/kafka-ingestion.md) or [Kin
On top of those requirements, allocating more task slots in your cluster is a good idea, so that you have free task
slots available for other tasks, such as [compaction tasks](../data-management/compaction.md).
-###### Hadoop ingestion
-
-If you are only using [Hadoop-based batch ingestion](../ingestion/hadoop.md) with no other ingestion types, you can lower the amount of resources allocated per Task. Batch ingestion tasks do not need to answer queries, and the bulk of the ingestion workload will be executed on the Hadoop cluster, so the Tasks do not require much resources.
-
###### Parallel native ingestion
If you are using [parallel native batch ingestion](../ingestion/native-batch.md), allocating more available task slots is a good idea and will allow greater ingestion concurrency.
diff --git a/docs/operations/other-hadoop.md b/docs/operations/other-hadoop.md
deleted file mode 100644
index 7d13a406c1e..00000000000
--- a/docs/operations/other-hadoop.md
+++ /dev/null
@@ -1,310 +0,0 @@
----
-id: other-hadoop
-title: "Working with different versions of Apache Hadoop"
----
-
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated and scheduled to be removed with Druid 37.0.0.
-
-We recommend one of Druid's other supported ingestion methods, such as [SQL-based ingestion](../multi-stage-query/index.md) or [MiddleManager-less ingestion using Kubernetes](../development/extensions-core/k8s-jobs.md)
-
-You must now explicitly opt-in to using the deprecated `index_hadoop` task type. To opt-in, set `druid.indexer.task.allowHadoopTaskExecution` to `true` in your `common.runtime.properties` file. For more information, see [#18239](https://github.com/apache/druid/pull/18239).
-
-:::
-
-
-Apache Druid can interact with Hadoop in two ways:
-
-1. [Use HDFS for deep storage](../development/extensions-core/hdfs.md) using the druid-hdfs-storage extension.
-2. [Batch-load data from Hadoop](../ingestion/hadoop.md) using Map/Reduce jobs.
-
-These are not necessarily linked together; you can load data with Hadoop jobs into a non-HDFS deep storage (like S3),
-and you can use HDFS for deep storage even if you're loading data from streams rather than using Hadoop jobs.
-
-For best results, use these tips when configuring Druid to interact with your favorite Hadoop distribution.
-
-## Tip #1: Place Hadoop XMLs on Druid classpath
-
-Place your Hadoop configuration XMLs (core-site.xml, hdfs-site.xml, yarn-site.xml, mapred-site.xml) on the classpath
-of your Druid processes. You can do this by copying them into `conf/druid/_common/core-site.xml`,
-`conf/druid/_common/hdfs-site.xml`, and so on. This allows Druid to find your Hadoop cluster and properly submit jobs.
-
-## Tip #2: Classloader modification on Hadoop (Map/Reduce jobs only)
-
-Druid uses a number of libraries that are also likely present on your Hadoop cluster, and if these libraries conflict,
-your Map/Reduce jobs can fail. This problem can be avoided by enabling classloader isolation using the Hadoop job
-property `mapreduce.job.classloader = true`. This instructs Hadoop to use a separate classloader for Druid dependencies
-and for Hadoop's own dependencies.
-
-If your version of Hadoop does not support this functionality, you can also try setting the property
-`mapreduce.job.user.classpath.first = true`. This instructs Hadoop to prefer loading Druid's version of a library when
-there is a conflict.
-
-Generally, you should only set one of these parameters, not both.
-
-These properties can be set in either one of the following ways:
-
-- Using the task definition, e.g. add `"mapreduce.job.classloader": "true"` to the `jobProperties` of the `tuningConfig` of your indexing task (see the [Hadoop batch ingestion documentation](../ingestion/hadoop.md)).
-- Using system properties, e.g. on the Middle Manager set `druid.indexer.runner.javaOpts=... -Dhadoop.mapreduce.job.classloader=true` in [Middle Manager configuration](../configuration/index.md#middle-manager-configuration).
-
-### Overriding specific classes
-
-When `mapreduce.job.classloader = true`, it is also possible to specifically define which classes should be loaded from the hadoop system classpath and which should be loaded from job-supplied JARs.
-
-This is controlled by defining class inclusion/exclusion patterns in the `mapreduce.job.classloader.system.classes` property in the `jobProperties` of `tuningConfig`.
-
-For example, some community members have reported version incompatibility errors with the Validator class:
-
-```
-Error: java.lang.ClassNotFoundException: javax.validation.Validator
-```
-
-The following `jobProperties` excludes `javax.validation.` classes from being loaded from the system classpath, while including those from `java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop.`.
-
-```
-"jobProperties": {
- "mapreduce.job.classloader": "true",
- "mapreduce.job.classloader.system.classes": "-javax.validation.,java.,javax.,org.apache.commons.logging.,org.apache.log4j.,org.apache.hadoop."
-}
-```
-
-[mapred-default.xml](https://hadoop.apache.org/docs/stable/hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml) documentation contains more information about this property.
-
-## Tip #3: Use specific versions of Hadoop libraries
-
-Druid loads Hadoop client libraries from two different locations. Each set of libraries is loaded in an isolated
-classloader.
-
-1. HDFS deep storage uses jars from `extensions/druid-hdfs-storage/` to read and write Druid data on HDFS.
-2. Batch ingestion uses jars from `hadoop-dependencies/` to submit Map/Reduce jobs (location customizable via the
-`druid.extensions.hadoopDependenciesDir` runtime property; see [Configuration](../configuration/index.md#extensions)).
-
-The default version of the Hadoop client bundled with Druid is `3.3.6`. This works with
-many Hadoop distributions (the version does not necessarily need to match), but if you run into issues, you can instead
-have Druid load libraries that exactly match your distribution. To do this, either copy the jars from your Hadoop
-cluster, or use the `pull-deps` tool to download the jars from a Maven repository.
-
-### Preferred: Load using Druid's standard mechanism
-
-If you have issues with HDFS deep storage, you can switch your Hadoop client libraries by recompiling the
-druid-hdfs-storage extension using an alternate version of the Hadoop client libraries. You can do this by editing
-the main Druid pom.xml and rebuilding the distribution by running `mvn package`.
-
-If you have issues with Map/Reduce jobs, you can switch your Hadoop client libraries without rebuilding Druid. You can
-do this by adding a new set of libraries to the `hadoop-dependencies/` directory (or another directory specified by
-druid.extensions.hadoopDependenciesDir) and then using `hadoopDependencyCoordinates` in the
-[Hadoop Index Task](../ingestion/hadoop.md) to specify the Hadoop dependencies you want Druid to load.
-
-Example:
-
-Suppose you specify `druid.extensions.hadoopDependenciesDir=/usr/local/druid_tarball/hadoop-dependencies`, and you have downloaded
-`hadoop-client` 2.3.0 and 2.4.0, either by copying them from your Hadoop cluster or by using `pull-deps` to download
-the jars from a Maven repository. Then underneath `hadoop-dependencies`, your jars should look like this:
-
-```
-hadoop-dependencies/
-โโโ hadoop-client
- โโโ 2.3.0
- โย ย โโโ activation-1.1.jar
- โย ย โโโ avro-1.7.4.jar
- โย ย โโโ commons-beanutils-1.7.0.jar
- โย ย โโโ commons-beanutils-core-1.8.0.jar
- โย ย โโโ commons-cli-1.2.jar
- โย ย โโโ commons-codec-1.4.jar
- ..... lots of jars
- โโโ 2.4.0
- โโโ activation-1.1.jar
- โโโ avro-1.7.4.jar
- โโโ commons-beanutils-1.7.0.jar
- โโโ commons-beanutils-core-1.8.0.jar
- โโโ commons-cli-1.2.jar
- โโโ commons-codec-1.4.jar
- ..... lots of jars
-```
-
-As you can see, under `hadoop-client`, there are two sub-directories, each denotes a version of `hadoop-client`.
-
-Next, use `hadoopDependencyCoordinates` in [Hadoop Index Task](../ingestion/hadoop.md) to specify the Hadoop dependencies you want Druid to load.
-
-For example, in your Hadoop Index Task spec file, you can write:
-
-`"hadoopDependencyCoordinates": ["org.apache.hadoop:hadoop-client:2.4.0"]`
-
-This instructs Druid to load hadoop-client 2.4.0 when processing the task. What happens behind the scene is that Druid first looks for a folder
-called `hadoop-client` underneath `druid.extensions.hadoopDependenciesDir`, then looks for a folder called `2.4.0`
-underneath `hadoop-client`, and upon successfully locating these folders, hadoop-client 2.4.0 is loaded.
-
-### Alternative: Append your Hadoop jars to the Druid classpath
-
-You can also load Hadoop client libraries in Druid's main classloader, rather than an isolated classloader. This
-mechanism is relatively easy to reason about, but it also means that you have to ensure that all dependency jars on the
-classpath are compatible. That is, Druid makes no provisions while using this method to maintain class loader isolation
-so you must make sure that the jars on your classpath are mutually compatible.
-
-1. Set `druid.indexer.task.defaultHadoopCoordinates=[]`. By setting this to an empty list, Druid will not load any other Hadoop dependencies except the ones specified in the classpath.
-2. Append your Hadoop jars to Druid's classpath. Druid will load them into the system.
-
-## Notes on specific Hadoop distributions
-
-If the tips above do not solve any issues you are having with HDFS deep storage or Hadoop batch indexing, you may
-have luck with one of the following suggestions contributed by the Druid community.
-
-### CDH
-
-Members of the community have reported dependency conflicts between the version of Jackson used in CDH and Druid when running a Mapreduce job like:
-
-```
-java.lang.VerifyError: class com.fasterxml.jackson.datatype.guava.deser.HostAndPortDeserializer overrides final method deserialize.(Lcom/fasterxml/jackson/core/JsonParser;Lcom/fasterxml/jackson/databind/DeserializationContext;)Ljava/lang/Object;
-```
-
-**Preferred workaround**
-
-First, try the tip under "Classloader modification on Hadoop" above. More recent versions of CDH have been reported to
-work with the classloader isolation option (`mapreduce.job.classloader = true`).
-
-**Alternate workaround - 1**
-
-You can try editing Druid's pom.xml dependencies to match the version of Jackson in your Hadoop version and recompile Druid.
-
-For more about building Druid, please see [Building Druid](../development/build.md).
-
-**Alternate workaround - 2**
-
-Another workaround solution is to build a custom fat jar of Druid using [sbt](http://www.scala-sbt.org/), which manually excludes all the conflicting Jackson dependencies, and then put this fat jar in the classpath of the command that starts Overlord indexing service. To do this, please follow the following steps.
-
-(1) Download and install sbt.
-
-(2) Make a new directory named 'druid_build'.
-
-(3) Cd to 'druid_build' and create the build.sbt file with the content [here](./use_sbt_to_build_fat_jar.md).
-
-You can always add more building targets or remove the ones you don't need.
-
-(4) In the same directory create a new directory named 'project'.
-
-(5) Put the druid source code into 'druid_build/project'.
-
-(6) Create a file 'druid_build/project/assembly.sbt' with content as follows.
-```
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0")
-```
-
-(7) In the 'druid_build' directory, run 'sbt assembly'.
-
-(8) In the 'druid_build/target/scala-2.10' folder, you will find the fat jar you just build.
-
-(9) Make sure the jars you've uploaded has been completely removed. The HDFS directory is by default '/tmp/druid-indexing/classpath'.
-
-(10) Include the fat jar in the classpath when you start the indexing service. Make sure you've removed 'lib/*' from your classpath because now the fat jar includes all you need.
-
-**Alternate workaround - 3**
-
-If sbt is not your choice, you can also use `maven-shade-plugin` to make a fat jar: relocation all Jackson packages will resolve it too. In this way, druid will not be affected by Jackson library embedded in hadoop. Please follow the steps below:
-
-(1) Add all extensions you needed to `services/pom.xml` like
-
- ```xml
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-avro-extensions</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-parquet-extensions</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-hdfs-storage</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>mysql-metadata-storage</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- ```
-
-(2) Shade Jackson packages and assemble a fat jar.
-
-```xml
-<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <outputFile>
- ${project.build.directory}/${project.artifactId}-${project.version}-selfcontained.jar
- </outputFile>
- <relocations>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>shade.com.fasterxml.jackson</shadedPattern>
- </relocation>
- </relocations>
- <artifactSet>
- <includes>
- <include>*:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
-```
-
-Copy out `services/target/xxxxx-selfcontained.jar` after `mvn install` in project root for further usage.
-
-(3) run hadoop indexer (post an indexing task is not possible now) as below. `lib` is not needed anymore. As hadoop indexer is a standalone tool, you don't have to replace the jars of your running services:
-
-```bash
-java -Xmx32m \
- -Dfile.encoding=UTF-8 -Duser.timezone=UTC \
- -classpath config/hadoop:config/overlord:config/_common:$SELF_CONTAINED_JAR:$HADOOP_DISTRIBUTION/etc/hadoop \
- -Djava.security.krb5.conf=$KRB5 \
- org.apache.druid.cli.Main index hadoop \
- $config_path
-```
diff --git a/docs/operations/pull-deps.md b/docs/operations/pull-deps.md
index b78d7c53288..39c85a1ba5f 100644
--- a/docs/operations/pull-deps.md
+++ b/docs/operations/pull-deps.md
@@ -31,17 +31,9 @@ title: "pull-deps tool"
Extension coordinate to pull down, followed by a maven coordinate, e.g. org.apache.druid.extensions:mysql-metadata-storage
-`-h` or `--hadoop-coordinate` (Can be specified multiply times)
-
-Apache Hadoop dependency to pull down, followed by a maven coordinate, e.g. org.apache.hadoop:hadoop-client:2.4.0
-
-`--no-default-hadoop`
-
-Don't pull down the default hadoop coordinate, i.e., org.apache.hadoop:hadoop-client:2.3.0. If `-h` option is supplied, then default hadoop coordinate will not be downloaded.
-
`--clean`
-Remove existing extension and hadoop dependencies directories before pulling down dependencies.
+Remove existing extension directories before pulling down dependencies.
`-l` or `--localRepository`
@@ -85,19 +77,19 @@ Set a password to connect to the proxy, this option is only required if the prox
To run `pull-deps`, you should
-1) Specify `druid.extensions.directory` and `druid.extensions.hadoopDependenciesDir`, these two properties tell `pull-deps` where to put extensions. If you don't specify them, default values will be used, see [Configuration](../configuration/index.md).
+1) Specify `druid.extensions.directory` which tells `pull-deps` where to put extensions. If you don't specify them, default values will be used, see [Configuration](../configuration/index.md).
-2) Tell `pull-deps` what to download using `-c` or `-h` option, which are followed by a maven coordinate.
+2) Tell `pull-deps` what to download using `-c` option, which are followed by a maven coordinate.
Example:
-Suppose you want to download ```mysql-metadata-storage``` and ```hadoop-client```(both 2.3.0 and 2.4.0) with a specific version, you can run `pull-deps` command with `-c org.apache.druid.extensions:mysql-metadata-storage:{{DRUIDVERSION}}`, `-h org.apache.hadoop:hadoop-client:2.3.0` and `-h org.apache.hadoop:hadoop-client:2.4.0`, an example command would be:
+Suppose you want to download ```mysql-metadata-storage``` with a specific version, you can run `pull-deps` command with `-c org.apache.druid.extensions:mysql-metadata-storage:{{DRUIDVERSION}}`, an example command would be:
```
-java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --clean -c org.apache.druid.extensions:mysql-metadata-storage:{{DRUIDVERSION}} -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0
+java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --clean -c org.apache.druid.extensions:mysql-metadata-storage:{{DRUIDVERSION}}
```
-Because `--clean` is supplied, this command will first remove the directories specified at `druid.extensions.directory` and `druid.extensions.hadoopDependenciesDir`, then recreate them and start downloading the extensions there. After finishing downloading, if you go to the extension directories you specified, you will see
+Because `--clean` is supplied, this command will first remove the directories specified at `druid.extensions.directory`, then recreate them and start downloading the extensions there. After finishing downloading, if you go to the extension directories you specified, you will see
```
tree extensions
@@ -106,32 +98,10 @@ extensions
โโโ mysql-metadata-storage-{{DRUIDVERSION}}.jar
```
-```
-tree hadoop-dependencies
-hadoop-dependencies/
-โโโ hadoop-client
- โโโ 2.3.0
- โย ย โโโ activation-1.1.jar
- โย ย โโโ avro-1.7.4.jar
- โย ย โโโ commons-beanutils-1.7.0.jar
- โย ย โโโ commons-beanutils-core-1.8.0.jar
- โย ย โโโ commons-cli-1.2.jar
- โย ย โโโ commons-codec-1.4.jar
- ..... lots of jars
- โโโ 2.4.0
- โโโ activation-1.1.jar
- โโโ avro-1.7.4.jar
- โโโ commons-beanutils-1.7.0.jar
- โโโ commons-beanutils-core-1.8.0.jar
- โโโ commons-cli-1.2.jar
- โโโ commons-codec-1.4.jar
- ..... lots of jars
-```
-
Note that if you specify `--defaultVersion`, you don't have to put version information in the coordinate. For example, if you want `mysql-metadata-storage` to use version `{{DRUIDVERSION}}`, you can change the command above to
```
-java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --defaultVersion {{DRUIDVERSION}} --clean -c org.apache.druid.extensions:mysql-metadata-storage -h org.apache.hadoop:hadoop-client:2.3.0 -h org.apache.hadoop:hadoop-client:2.4.0
+java -classpath "/my/druid/lib/*" org.apache.druid.cli.Main tools pull-deps --defaultVersion {{DRUIDVERSION}} --clean -c org.apache.druid.extensions:mysql-metadata-storage
```
:::info
diff --git a/docs/operations/segment-optimization.md b/docs/operations/segment-optimization.md
index 79dbf9db6b1..85c64a482a7 100644
--- a/docs/operations/segment-optimization.md
+++ b/docs/operations/segment-optimization.md
@@ -95,7 +95,7 @@ Once you find your segments need compaction, you can consider the below two opti
The Coordinator periodically submits [compaction tasks](../ingestion/tasks.md#compact) to re-index small segments.
To enable the automatic compaction, you need to configure it for each dataSource via Coordinator's dynamic configuration.
For more information, see [Automatic compaction](../data-management/automatic-compaction.md).
- - Running periodic Hadoop batch ingestion jobs and using a `dataSource`
+ - Running periodic batch ingestion jobs and using a `dataSource`
inputSpec to read from the segments generated by the Kafka indexing tasks. This might be helpful if you want to compact a lot of segments in parallel.
Details on how to do this can be found on the [Updating existing data](../data-management/update.md) section
of the data management page.
diff --git a/docs/operations/use_sbt_to_build_fat_jar.md b/docs/operations/use_sbt_to_build_fat_jar.md
deleted file mode 100644
index eeb58c9734e..00000000000
--- a/docs/operations/use_sbt_to_build_fat_jar.md
+++ /dev/null
@@ -1,127 +0,0 @@
----
-id: use_sbt_to_build_fat_jar
-title: "Content for build.sbt"
----
-
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-
-```scala
-libraryDependencies ++= Seq(
- "com.amazonaws" % "aws-java-sdk" % "1.9.23" exclude("common-logging", "common-logging"),
- "org.joda" % "joda-convert" % "1.7",
- "joda-time" % "joda-time" % "2.7",
- "org.apache.druid" % "druid" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid" % "druid-services" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid" % "druid-indexing-service" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid" % "druid-indexing-hadoop" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid.extensions" % "mysql-metadata-storage" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid.extensions" % "druid-s3-extensions" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid.extensions" % "druid-histogram" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "org.apache.druid.extensions" % "druid-hdfs-storage" % "0.8.1" excludeAll (
- ExclusionRule("org.ow2.asm"),
- ExclusionRule("com.fasterxml.jackson.core"),
- ExclusionRule("com.fasterxml.jackson.datatype"),
- ExclusionRule("com.fasterxml.jackson.dataformat"),
- ExclusionRule("com.fasterxml.jackson.jaxrs"),
- ExclusionRule("com.fasterxml.jackson.module")
- ),
- "com.fasterxml.jackson.core" % "jackson-annotations" % "2.3.0",
- "com.fasterxml.jackson.core" % "jackson-core" % "2.3.0",
- "com.fasterxml.jackson.core" % "jackson-databind" % "2.3.0",
- "com.fasterxml.jackson.datatype" % "jackson-datatype-guava" % "2.3.0",
- "com.fasterxml.jackson.datatype" % "jackson-datatype-joda" % "2.3.0",
- "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-base" % "2.3.0",
- "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider" % "2.3.0",
- "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-smile-provider" % "2.3.0",
- "com.fasterxml.jackson.module" % "jackson-module-jaxb-annotations" % "2.3.0",
- "com.sun.jersey" % "jersey-servlet" % "1.17.1",
- "mysql" % "mysql-connector-java" % "8.2.0",
- "org.scalatest" %% "scalatest" % "2.2.3" % "test",
- "org.mockito" % "mockito-core" % "1.10.19" % "test"
-)
-
-assemblyMergeStrategy in assembly := {
- case path if path contains "pom." => MergeStrategy.first
- case path if path contains "javax.inject.Named" => MergeStrategy.first
- case path if path contains "mime.types" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/impl/SimpleLog.class" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/impl/SimpleLog$1.class" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/impl/NoOpLog.class" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/LogFactory.class" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/LogConfigurationException.class" => MergeStrategy.first
- case path if path contains "org/apache/commons/logging/Log.class" => MergeStrategy.first
- case path if path contains "META-INF/jersey-module-version" => MergeStrategy.first
- case path if path contains ".properties" => MergeStrategy.first
- case path if path contains ".class" => MergeStrategy.first
- case x =>
- val oldStrategy = (assemblyMergeStrategy in assembly).value
- oldStrategy(x)
-}
-```
diff --git a/docs/querying/geo.md b/docs/querying/geo.md
index 56c4645896f..e5a9e0bba0d 100644
--- a/docs/querying/geo.md
+++ b/docs/querying/geo.md
@@ -40,45 +40,29 @@ Spatial dimensions are string columns that contain coordinates separated by a co
In the ingestion spec, you configure spatial dimensions in the `dimensionsSpec` object of the `dataSchema` component.
You can provide spatial dimensions in any of the [data formats](../ingestion/data-formats.md) supported by Druid.
-The following example shows an ingestion spec with a spatial dimension named `coordinates`, which is constructed from the input fields `x` and `y`:
+The following example shows dimension spec with a spatial dimension named `coordinates`, which is constructed from the input fields `x` and `y`:
```json
{
- "type": "hadoop",
- "dataSchema": {
- "dataSource": "DatasourceName",
- "parser": {
- "type": "string",
- "parseSpec": {
- "format": "json",
- "timestampSpec": {
- "column": "timestamp",
- "format": "auto"
- },
- "dimensionsSpec": {
- "dimensions": [
- {
- "type": "double",
- "name": "x"
- },
- {
- "type": "double",
- "name": "y"
- }
- ],
- "spatialDimensions": [
- {
- "dimName": "coordinates",
- "dims": [
- "x",
- "y"
- ]
- }
- ]
- }
- }
- }
+ "dimensions": [
+ {
+ "type": "double",
+ "name": "x"
+ },
+ {
+ "type": "double",
+ "name": "y"
+ }
+ ],
+ "spatialDimensions": [
+ {
+ "dimName": "coordinates",
+ "dims": [
+ "x",
+ "y"
+ ]
}
+ ]
}
```
diff --git a/docs/querying/multitenancy.md b/docs/querying/multitenancy.md
index bc70177d0fa..e303f4dfb06 100644
--- a/docs/querying/multitenancy.md
+++ b/docs/querying/multitenancy.md
@@ -43,7 +43,6 @@ and expiration rules.
Pros of shared datasources:
- Each datasource requires its own JVMs for realtime indexing.
-- Each datasource requires its own YARN resources for Hadoop batch jobs.
- Each datasource requires its own segment files on disk.
- For these reasons it can be wasteful to have a very large number of small datasources.
@@ -57,7 +56,7 @@ If your multitenant cluster uses shared datasources, most of your queries will l
dimension. These sorts of queries perform best when data is well-partitioned by tenant. There are a few ways to
accomplish this.
-With batch indexing, you can use [single-dimension partitioning](../ingestion/hadoop.md#single-dimension-range-partitioning)
+With batch indexing, you can use [multi-dimension partitioning](../ingestion/native-batch.md#multi-dimension-range-partitioning)
to partition your data by tenant_id. Druid always partitions by time first, but the secondary partition within each
time bucket will be on tenant_id.
diff --git a/docs/tutorials/cluster.md b/docs/tutorials/cluster.md
index cd435c5e1ce..baf2ef46e67 100644
--- a/docs/tutorials/cluster.md
+++ b/docs/tutorials/cluster.md
@@ -166,7 +166,6 @@ In the package, you should find:
* `bin/*` - scripts related to the [single-machine quickstart](index.md)
* `conf/druid/cluster/*` - template configurations for a clustered setup
* `extensions/*` - core Druid extensions
-* `hadoop-dependencies/*` - Druid Hadoop dependencies
* `lib/*` - libraries and dependencies for core Druid
* `quickstart/*` - files related to the [single-machine quickstart](index.md)
@@ -278,27 +277,6 @@ mapred-site.xml) on the classpath of your Druid processes. You can do this by co
Please see the [HDFS extension](../development/extensions-core/hdfs.md) documentation for more info.
-<a name="hadoop"></a>
-
-## Configure for connecting to Hadoop (optional)
-
-If you will be loading data from a Hadoop cluster, then at this point you should configure Druid to be aware
-of your cluster:
-
-- Update `druid.indexer.task.hadoopWorkingPath` in `conf/druid/cluster/middleManager/runtime.properties` to
-a path on HDFS that you'd like to use for temporary files required during the indexing process.
-`druid.indexer.task.hadoopWorkingPath=/tmp/druid-indexing` is a common choice.
-
-- Place your Hadoop configuration XMLs (core-site.xml, hdfs-site.xml, yarn-site.xml,
-mapred-site.xml) on the classpath of your Druid processes. You can do this by copying them into
-`conf/druid/cluster/_common/core-site.xml`, `conf/druid/cluster/_common/hdfs-site.xml`, and so on.
-
-Note that you don't need to use HDFS deep storage in order to load data from Hadoop. For example, if
-your cluster is running on Amazon Web Services, we recommend using S3 for deep storage even if you
-are loading data using Hadoop or Elastic MapReduce.
-
-For more info, please see the [Hadoop-based ingestion](../ingestion/hadoop.md) page.
-
## Configure Zookeeper connection
In a production cluster, we recommend using a dedicated ZK cluster in a quorum, deployed separately from the Druid servers.
diff --git a/docs/tutorials/tutorial-batch-hadoop.md b/docs/tutorials/tutorial-batch-hadoop.md
deleted file mode 100644
index a3995736885..00000000000
--- a/docs/tutorials/tutorial-batch-hadoop.md
+++ /dev/null
@@ -1,252 +0,0 @@
----
-id: tutorial-batch-hadoop
-title: Load batch data using Apache Hadoop
-sidebar_label: Load from Apache Hadoop
----
-
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-:::caution[Deprecated]
-
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-:::
-
-
-This tutorial shows you how to load data files into Apache Druid using a remote Hadoop cluster.
-
-For this tutorial, we'll assume that you've already completed the previous
-[batch ingestion tutorial](tutorial-batch.md) using Druid's native batch ingestion system and are using the
-automatic single-machine configuration as described in the [quickstart](../operations/single-server.md).
-
-## Install Docker
-
-This tutorial requires [Docker](https://docs.docker.com/install/) to be installed on the tutorial machine.
-
-Once the Docker install is complete, please proceed to the next steps in the tutorial.
-
-## Build the Hadoop docker image
-
-For this tutorial, we've provided a Dockerfile for a Hadoop 3.3.6 cluster, which we'll use to run the batch indexing task.
-
-This Dockerfile and related files are located at `quickstart/tutorial/hadoop/docker`.
-
-From the `apache-druid-{{DRUIDVERSION}}` package root, run the following commands to build a Docker image named "druid-hadoop-demo" with version tag "3.3.6":
-
-```bash
-cd quickstart/tutorial/hadoop/docker
-docker build -t druid-hadoop-demo:3.3.6 .
-```
-
-This will start building the Hadoop image. Once the image build is done, you should see the message `Successfully tagged druid-hadoop-demo:3.3.6` printed to the console.
-
-## Setup the Hadoop docker cluster
-
-### Create temporary shared directory
-
-We'll need a shared folder between the host and the Hadoop container for transferring some files.
-
-Let's create some folders under `/tmp`, we will use these later when starting the Hadoop container:
-
-```bash
-mkdir -p /tmp/shared
-```
-
-### Configure /etc/hosts
-
-On the host machine, add the following entry to `/etc/hosts`:
-
-```
-127.0.0.1 druid-hadoop-demo
-```
-
-### Start the Hadoop container
-
-Once the `/tmp/shared` folder has been created and the `etc/hosts` entry has been added, run the following command to start the Hadoop container.
-
-```bash
-docker run -it -h druid-hadoop-demo --name druid-hadoop-demo -p 2049:2049 -p 2122:2122 -p 8020-8042:8020-8042 -p 8088:8088 -p 8443:8443 -p 9000:9000 -p 9820:9820 -p 9860-9880:9860-9880 -p 10020:10020 -p 19888:19888 -p 34455:34455 -p 49707:49707 -p 50010:50010 -p 50020:50020 -p 50030:50030 -p 50060:50060 -p 50070:50070 -p 50075:50075 -p 50090:50090 -p 51111:51111 -v /tmp/shared:/shared druid-hadoop-demo:3.3.6 /etc/bootstrap.sh -bash
-```
-
-Once the container is started, your terminal will attach to a bash shell running inside the container:
-
-```bash
-Starting namenodes on [druid-hadoop-demo]
-Starting datanodes
-Starting secondary namenodes [druid-hadoop-demo]
-WARNING: YARN_CONF_DIR has been replaced by HADOOP_CONF_DIR. Using value of YARN_CONF_DIR.
-WARNING: YARN_OPTS has been replaced by HADOOP_OPTS. Using value of YARN_OPTS.
-Starting resourcemanager
-WARNING: YARN_OPTS has been replaced by HADOOP_OPTS. Using value of YARN_OPTS.
-Starting nodemanagers
-WARNING: YARN_OPTS has been replaced by HADOOP_OPTS. Using value of YARN_OPTS.
-localhost: WARNING: YARN_OPTS has been replaced by HADOOP_OPTS. Using value of YARN_OPTS.
-WARNING: YARN_CONF_DIR has been replaced by HADOOP_CONF_DIR. Using value of YARN_CONF_DIR.
-WARNING: YARN_OPTS has been replaced by HADOOP_OPTS. Using value of YARN_OPTS.
-WARNING: Use of this script to start the MR JobHistory daemon is deprecated.
-WARNING: Attempting to execute replacement "mapred --daemon start" instead.
- * initialize hdfs for first run
-bash-4.1#
-```
-
-The `Unable to load native-hadoop library for your platform... using builtin-java classes where applicable` warning messages can be safely ignored.
-
-#### Accessing the Hadoop container shell
-
-To open another shell to the Hadoop container, run the following command:
-
-```
-docker exec -it druid-hadoop-demo bash
-```
-
-### Test data
-
-The startup script `bootstrap.sh`:
-* creates the necessary directories
-* loads an input file to HDFS
-* places the hadoop configuration into the shared volume as `hadoop-conf.tgz`
-
-## Configure Druid to use Hadoop
-
-Some additional steps are needed to configure the Druid cluster for Hadoop batch indexing.
-
-### Provide Hadoop configuration for Druid
-
-From the Hadoop container's shell, run the following command to copy the Hadoop .xml configuration files to the shared folder:
-
-```bash
-cp /usr/local/hadoop/etc/hadoop/*.xml /shared/hadoop_xml
-```
-
-From the host machine, run the following, where `PATH_TO_DRUID` is replaced by the path to the Druid package.
-
-```bash
-cd $PATH_TO_DRUID
-mkdir -p conf/druid/single-server/micro-quickstart/_common/hadoop-xml
-tar xzf /tmp/shared/hadoop-conf.tgz -C conf/druid/single-server/micro-quickstart/_common/hadoop-xml
-```
-
-### Update Druid segment and log storage
-
-In your favorite text editor, open `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties`, and make the following edits:
-
-#### Disable local deep storage and enable HDFS deep storage
-
-```
-#
-# Deep storage
-#
-
-# For local disk (only viable in a cluster if this is a network mount):
-#druid.storage.type=local
-#druid.storage.storageDirectory=var/druid/segments
-
-# For HDFS:
-druid.storage.type=hdfs
-druid.storage.storageDirectory=/druid/segments
-```
-
-
-#### Disable local log storage and enable HDFS log storage
-
-```
-#
-# Indexing service logs
-#
-
-# For local disk (only viable in a cluster if this is a network mount):
-#druid.indexer.logs.type=file
-#druid.indexer.logs.directory=var/druid/indexing-logs
-
-# For HDFS:
-druid.indexer.logs.type=hdfs
-druid.indexer.logs.directory=/druid/indexing-logs
-
-```
-
-### Restart Druid cluster
-
-Once the Hadoop .xml files have been copied to the Druid cluster and the segment/log storage configuration has been updated to use HDFS, the Druid cluster needs to be restarted for the new configurations to take effect.
-
-If the cluster is still running, CTRL-C to terminate it if running - and start it with:
-```
-bin/start-druid -c conf/druid/single-server/micro-quickstart
-```
-
-## Load batch data
-
-We've included a sample of Wikipedia edits from September 12, 2015 to get you started.
-
-To load this data into Druid, you can submit an *ingestion task* pointing to the file. We've included
-a task that loads the `wikiticker-2015-09-12-sampled.json.gz` file included in the archive.
-
-Let's submit the `wikipedia-index-hadoop3.json` task:
-
-```bash
-bin/post-index-task --file quickstart/tutorial/wikipedia-index-hadoop3.json --url http://localhost:8081
-```
-
-## Querying your data
-
-After the data load is complete, please follow the [query tutorial](../tutorials/tutorial-query.md) to run some example queries on the newly loaded data.
-
-## Cleanup
-
-This tutorial is only meant to be used together with the [query tutorial](../tutorials/tutorial-query.md).
-
-If you wish to go through any of the other tutorials, you will need to:
-* Shut down the cluster and reset the cluster state by removing the contents of the `var` directory under the druid package.
-* Revert the deep storage and task storage config back to local types in `conf/druid/single-server/micro-quickstart/_common/common.runtime.properties`
-* Restart the cluster
-
-This is necessary because the other ingestion tutorials will write to the same "wikipedia" datasource, and later tutorials expect the cluster to use local deep storage.
-
-Example reverted config:
-
-```
-#
-# Deep storage
-#
-
-# For local disk (only viable in a cluster if this is a network mount):
-druid.storage.type=local
-druid.storage.storageDirectory=var/druid/segments
-
-# For HDFS:
-#druid.storage.type=hdfs
-#druid.storage.storageDirectory=/druid/segments
-
-#
-# Indexing service logs
-#
-
-# For local disk (only viable in a cluster if this is a network mount):
-druid.indexer.logs.type=file
-druid.indexer.logs.directory=var/druid/indexing-logs
-
-# For HDFS:
-#druid.indexer.logs.type=hdfs
-#druid.indexer.logs.directory=/druid/indexing-logs
-
-```
-
-## Further reading
-
-For more information on loading batch data with Hadoop, please see [the Hadoop batch ingestion documentation](../ingestion/hadoop.md).
diff --git a/docs/tutorials/tutorial-kerberos-hadoop.md b/docs/tutorials/tutorial-kerberos-hadoop.md
index 0ec798e34a3..9a033cd7965 100644
--- a/docs/tutorials/tutorial-kerberos-hadoop.md
+++ b/docs/tutorials/tutorial-kerberos-hadoop.md
@@ -23,19 +23,13 @@ sidebar_label: Kerberized HDFS deep storage
~ under the License.
-->
-:::caution[Deprecated]
-Hadoop-based ingestion is deprecated. For more information, see the [upgrade notes](../release-info/upgrade-notes.md#hadoop-based-ingestion).
-
-:::
-
-
-## Hadoop Setup
+## Setup
Following are the configurations files required to be copied over to Druid conf folders:
-1. For HDFS as a deep storage, hdfs-site.xml, core-site.xml
-2. For ingestion, mapred-site.xml, yarn-site.xml
+1. hdfs-site.xml
+2. core-site.xml
### HDFS Folders and permissions
@@ -71,15 +65,6 @@ druid.storage.storageDirectory=/druid/segments
# OR
# druid.storage.storageDirectory=/apps/druid/segments
-#
-# Indexing service logs
-#
-
-# For HDFS:
-druid.indexer.logs.type=hdfs
-druid.indexer.logs.directory=/druid/indexing-logs
-# OR
-# druid.storage.storageDirectory=/apps/druid/indexing-logs
```
Note: Comment out Local storage and S3 Storage parameters in the file
@@ -92,23 +77,12 @@ Also include hdfs-storage core extension to `conf/druid/_common/common.runtime.p
#
druid.extensions.directory=dist/druid/extensions
-druid.extensions.hadoopDependenciesDir=dist/druid/hadoop-dependencies
druid.extensions.loadList=["mysql-metadata-storage", "druid-hdfs-storage", "druid-kerberos"]
```
-### Hadoop Jars
-
-Ensure that Druid has necessary jars to support the Hadoop version.
-
-Find the hadoop version using command, `hadoop version`
-
-In case there is other software used with hadoop, like `WanDisco`, ensure that
-1. the necessary libraries are available
-2. add the requisite extensions to `druid.extensions.loadlist` in `conf/druid/_common/common.runtime.properties`
-
### Kerberos setup
-Create a headless keytab which would have access to the druid data and index.
+Create a headless keytab which would have access to the druid data.
Edit conf/druid/_common/common.runtime.properties and add the following properties:
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 1cc344d4ca5..09d411fc1c1 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -902,6 +902,27 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- needed by iceberg tests -->
+ <dependency>
+ <groupId>org.apache.parquet</groupId>
+ <artifactId>parquet-avro</artifactId>
+ <version>1.15.2</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-pool</groupId>
+ <artifactId>commons-pool</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>it.unimi.dsi</groupId>
+ <artifactId>fastutil</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
<build>
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedSSLAuthResource.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedSSLAuthResource.java
index 900272ce3fa..c13240bf4a3 100644
--- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedSSLAuthResource.java
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/EmbeddedSSLAuthResource.java
@@ -34,9 +34,9 @@ import java.io.IOException;
* <ul>
* <li>Enable TLS port on the Druid services</li>
* <li>Generate client certificates using script
- * {@code integration-tests/docker/tls/generate-client-certs-and-keystores.sh}</li>
+ * {@code tls/generate-client-certs-and-keystores.sh}</li>
* <li>Generate server truststore using script
- * {@code integration-tests/docker/tls/generate-server-certs-and-keystores.sh}</li>
+ * {@code tls/generate-server-certs-and-keystores.sh}</li>
* <li>Keep the generated certificates in the {@code TestFolder} used by the cluster</li>
* <li>Configure the cluster with the appropriate keystore/truststore paths.</li>
* </ul>
@@ -91,7 +91,7 @@ public class EmbeddedSSLAuthResource implements EmbeddedResource
private void copyScriptsToDirectory(File targetDir)
{
try {
- FileUtils.copyDirectory(new File("../integration-tests/docker/tls"), targetDir);
+ FileUtils.copyDirectory(new File("./tls"), targetDir);
}
catch (IOException e) {
throw new RuntimeException(e);
diff --git a/integration-tests/docker/tls/generate-client-certs-and-keystores.sh b/embedded-tests/tls/generate-client-certs-and-keystores.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-client-certs-and-keystores.sh
rename to embedded-tests/tls/generate-client-certs-and-keystores.sh
diff --git a/integration-tests/docker/tls/generate-expired-client-cert.sh b/embedded-tests/tls/generate-expired-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-expired-client-cert.sh
rename to embedded-tests/tls/generate-expired-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-good-client-cert.sh b/embedded-tests/tls/generate-good-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-good-client-cert.sh
rename to embedded-tests/tls/generate-good-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh b/embedded-tests/tls/generate-incorrect-hostname-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-incorrect-hostname-client-cert.sh
rename to embedded-tests/tls/generate-incorrect-hostname-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh b/embedded-tests/tls/generate-invalid-intermediate-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-invalid-intermediate-client-cert.sh
rename to embedded-tests/tls/generate-invalid-intermediate-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-root-certs.sh b/embedded-tests/tls/generate-root-certs.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-root-certs.sh
rename to embedded-tests/tls/generate-root-certs.sh
diff --git a/integration-tests/docker/tls/generate-server-certs-and-keystores.sh b/embedded-tests/tls/generate-server-certs-and-keystores.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-server-certs-and-keystores.sh
rename to embedded-tests/tls/generate-server-certs-and-keystores.sh
diff --git a/integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh b/embedded-tests/tls/generate-to-be-revoked-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-to-be-revoked-client-cert.sh
rename to embedded-tests/tls/generate-to-be-revoked-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-untrusted-root-client-cert.sh b/embedded-tests/tls/generate-untrusted-root-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-untrusted-root-client-cert.sh
rename to embedded-tests/tls/generate-untrusted-root-client-cert.sh
diff --git a/integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh b/embedded-tests/tls/generate-valid-intermediate-client-cert.sh
similarity index 100%
rename from integration-tests/docker/tls/generate-valid-intermediate-client-cert.sh
rename to embedded-tests/tls/generate-valid-intermediate-client-cert.sh
diff --git a/integration-tests/docker/tls/root.cnf b/embedded-tests/tls/root.cnf
similarity index 100%
rename from integration-tests/docker/tls/root.cnf
rename to embedded-tests/tls/root.cnf
diff --git a/integration-tests/docker/tls/root2.cnf b/embedded-tests/tls/root2.cnf
similarity index 100%
rename from integration-tests/docker/tls/root2.cnf
rename to embedded-tests/tls/root2.cnf
diff --git a/integration-tests/docker/tls/set-docker-host-ip.sh b/embedded-tests/tls/set-docker-host-ip.sh
similarity index 100%
rename from integration-tests/docker/tls/set-docker-host-ip.sh
rename to embedded-tests/tls/set-docker-host-ip.sh
diff --git a/examples/conf/druid/auto/_common/common.runtime.properties b/examples/conf/druid/auto/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/auto/_common/common.runtime.properties
+++ b/examples/conf/druid/auto/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/auto/indexer/runtime.properties b/examples/conf/druid/auto/indexer/runtime.properties
index 6d5f718b580..4d9a28ed4c5 100644
--- a/examples/conf/druid/auto/indexer/runtime.properties
+++ b/examples/conf/druid/auto/indexer/runtime.properties
@@ -29,6 +29,3 @@ druid.worker.baseTaskDirs=[\"var/druid/task\"]
# Processing threads and buffers on Indexer
# Determined automatically based on available memory. For details on how to manually set parameters:
# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/auto/middleManager/runtime.properties b/examples/conf/druid/auto/middleManager/runtime.properties
index 744b2fdb3b6..d57f7e07c8a 100644
--- a/examples/conf/druid/auto/middleManager/runtime.properties
+++ b/examples/conf/druid/auto/middleManager/runtime.properties
@@ -30,6 +30,3 @@ druid.worker.baseTaskDirs=[\"var/druid/task\"]
# Processing threads and buffers on Peons
# Determined automatically based on available memory. For details on how to manually set parameters:
# https://druid.apache.org/docs/latest/operations/basic-cluster-tuning.html#guidelines-for-processing-threads-and-buffers
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/cluster/_common/common.runtime.properties b/examples/conf/druid/cluster/_common/common.runtime.properties
index 383d03e5709..b3564b50a5a 100644
--- a/examples/conf/druid/cluster/_common/common.runtime.properties
+++ b/examples/conf/druid/cluster/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/cluster/data/indexer/runtime.properties b/examples/conf/druid/cluster/data/indexer/runtime.properties
index 7c6018c356e..bac0732372a 100644
--- a/examples/conf/druid/cluster/data/indexer/runtime.properties
+++ b/examples/conf/druid/cluster/data/indexer/runtime.properties
@@ -33,6 +33,3 @@ druid.server.http.numThreads=60
druid.processing.numMergeBuffers=2
druid.processing.buffer.sizeBytes=100MiB
druid.processing.numThreads=4
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/cluster/data/middleManager/runtime.properties b/examples/conf/druid/cluster/data/middleManager/runtime.properties
index abf31d538ba..00b577e442f 100644
--- a/examples/conf/druid/cluster/data/middleManager/runtime.properties
+++ b/examples/conf/druid/cluster/data/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=60
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/large/_common/common.runtime.properties b/examples/conf/druid/single-server/large/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/single-server/large/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/large/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/single-server/large/middleManager/runtime.properties b/examples/conf/druid/single-server/large/middleManager/runtime.properties
index 1045054e721..edc249a3232 100644
--- a/examples/conf/druid/single-server/large/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/large/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=60
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/medium/_common/common.runtime.properties b/examples/conf/druid/single-server/medium/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/single-server/medium/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/medium/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/single-server/medium/middleManager/runtime.properties b/examples/conf/druid/single-server/medium/middleManager/runtime.properties
index abf31d538ba..00b577e442f 100644
--- a/examples/conf/druid/single-server/medium/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/medium/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=60
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/micro-quickstart/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties
index 9cf3d5928d9..20563a918cf 100644
--- a/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/micro-quickstart/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=12
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties b/examples/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/nano-quickstart/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties
index f2ddfa3d8fa..4a8fc01b6bc 100644
--- a/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/nano-quickstart/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=6
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=25MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/small/_common/common.runtime.properties b/examples/conf/druid/single-server/small/_common/common.runtime.properties
index 54bf8a2ca9b..5a0a2bfd158 100644
--- a/examples/conf/druid/single-server/small/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/small/_common/common.runtime.properties
@@ -26,10 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
#
# Hostname
diff --git a/examples/conf/druid/single-server/small/middleManager/runtime.properties b/examples/conf/druid/single-server/small/middleManager/runtime.properties
index ce1a3c7f8bf..6a5c0d934fd 100644
--- a/examples/conf/druid/single-server/small/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/small/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=50
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties b/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties
index 54bf8a2ca9b..abd6c91aa65 100644
--- a/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties
+++ b/examples/conf/druid/single-server/xlarge/_common/common.runtime.properties
@@ -26,11 +26,6 @@
# More info: https://druid.apache.org/docs/latest/operations/including-extensions.html
druid.extensions.loadList=["druid-hdfs-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-parquet-extensions"]
-# If you have a different version of Hadoop, place your Hadoop client jar files in your hadoop-dependencies directory
-# and uncomment the line below to point to your directory.
-#druid.extensions.hadoopDependenciesDir=/my/dir/hadoop-dependencies
-
-
#
# Hostname
#
diff --git a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties
index a3411a06a7f..9e3159e00a5 100644
--- a/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties
+++ b/examples/conf/druid/single-server/xlarge/middleManager/runtime.properties
@@ -34,6 +34,3 @@ druid.server.http.numThreads=60
druid.indexer.fork.property.druid.processing.numMergeBuffers=2
druid.indexer.fork.property.druid.processing.buffer.sizeBytes=100MiB
druid.indexer.fork.property.druid.processing.numThreads=1
-
-# Hadoop indexing
-druid.indexer.task.hadoopWorkingPath=var/druid/hadoop-tmp
diff --git a/examples/quickstart/tutorial/wikipedia-index-hadoop3.json b/examples/quickstart/tutorial/wikipedia-index-hadoop3.json
deleted file mode 100644
index 30b60ffbb15..00000000000
--- a/examples/quickstart/tutorial/wikipedia-index-hadoop3.json
+++ /dev/null
@@ -1,81 +0,0 @@
-{
- "type" : "index_hadoop",
- "spec" : {
- "dataSchema" : {
- "dataSource" : "wikipedia",
- "parser" : {
- "type" : "hadoopyString",
- "parseSpec" : {
- "format" : "json",
- "dimensionsSpec" : {
- "dimensions" : [
- "channel",
- "cityName",
- "comment",
- "countryIsoCode",
- "countryName",
- "isAnonymous",
- "isMinor",
- "isNew",
- "isRobot",
- "isUnpatrolled",
- "metroCode",
- "namespace",
- "page",
- "regionIsoCode",
- "regionName",
- "user",
- { "name": "added", "type": "long" },
- { "name": "deleted", "type": "long" },
- { "name": "delta", "type": "long" }
- ]
- },
- "timestampSpec" : {
- "format" : "auto",
- "column" : "time"
- }
- }
- },
- "metricsSpec" : [],
- "granularitySpec" : {
- "type" : "uniform",
- "segmentGranularity" : "day",
- "queryGranularity" : "none",
- "intervals" : ["2015-09-12/2015-09-13"],
- "rollup" : false
- }
- },
- "ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "static",
- "paths" : "/quickstart/wikiticker-2015-09-12-sampled.json.gz"
- }
- },
- "tuningConfig" : {
- "type" : "hadoop",
- "partitionsSpec" : {
- "type" : "hashed",
- "targetPartitionSize" : 5000000
- },
- "forceExtendableShardSpecs" : true,
- "jobProperties" : {
- "fs.default.name" : "hdfs://druid-hadoop-demo:9000",
- "fs.defaultFS" : "hdfs://druid-hadoop-demo:9000",
- "dfs.datanode.address" : "druid-hadoop-demo",
- "dfs.client.use.datanode.hostname" : "true",
- "dfs.datanode.use.datanode.hostname" : "true",
- "yarn.resourcemanager.hostname" : "druid-hadoop-demo",
- "yarn.nodemanager.vmem-check-enabled" : "false",
- "mapreduce.map.java.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
- "mapreduce.job.user.classpath.first" : "true",
- "mapreduce.reduce.java.opts" : "-Duser.timezone=UTC -Dfile.encoding=UTF-8",
- "mapreduce.map.memory.mb" : 1024,
- "mapreduce.reduce.memory.mb" : 1024,
- "yarn.app.mapreduce.am.command-opts" : "--add-opens=java.base/java.lang=ALL-UNNAMED",
- "mapreduce.job.classloader" : "true"
- }
- }
- },
- "hadoopDependencyCoordinates": ["org.apache.hadoop:hadoop-client-api:3.3.6","org.apache.hadoop:hadoop-client-runtime:3.3.6"]
-}
diff --git a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java
index b824232c653..177d4dc1965 100644
--- a/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java
+++ b/extensions-contrib/aliyun-oss-extensions/src/main/java/org/apache/druid/storage/aliyun/OssDataSegmentPusher.java
@@ -21,10 +21,8 @@ package org.apache.druid.storage.aliyun;
import com.aliyun.oss.OSS;
import com.aliyun.oss.OSSException;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.inject.Inject;
-import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.segment.SegmentUtils;
import org.apache.druid.segment.loading.DataSegmentPusher;
@@ -34,7 +32,6 @@ import org.apache.druid.utils.CompressionUtils;
import java.io.File;
import java.io.IOException;
import java.net.URI;
-import java.util.List;
import java.util.Map;
public class OssDataSegmentPusher implements DataSegmentPusher
@@ -54,25 +51,6 @@ public class OssDataSegmentPusher implements DataSegmentPusher
this.config = config;
}
- @Override
- public String getPathForHadoop()
- {
- return StringUtils.format("%s/%s", config.getBucket(), config.getPrefix());
- }
-
- @Deprecated
- @Override
- public String getPathForHadoop(String dataSource)
- {
- return getPathForHadoop();
- }
-
- @Override
- public List<String> getAllowedPropertyPrefixesForHadoop()
- {
- return ImmutableList.of("druid.oss");
- }
-
@Override
public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath)
throws IOException
diff --git a/extensions-contrib/cassandra-storage/src/main/java/org/apache/druid/storage/cassandra/CassandraDataSegmentPusher.java b/extensions-contrib/cassandra-storage/src/main/java/org/apache/druid/storage/cassandra/CassandraDataSegmentPusher.java
index b96902c3365..126467c10a9 100644
--- a/extensions-contrib/cassandra-storage/src/main/java/org/apache/druid/storage/cassandra/CassandraDataSegmentPusher.java
+++ b/extensions-contrib/cassandra-storage/src/main/java/org/apache/druid/storage/cassandra/CassandraDataSegmentPusher.java
@@ -60,19 +60,6 @@ public class CassandraDataSegmentPusher extends CassandraStorage implements Data
this.jsonMapper = jsonMapper;
}
- @Override
- public String getPathForHadoop()
- {
- throw new UnsupportedOperationException("Cassandra storage does not support indexing via Hadoop");
- }
-
- @Deprecated
- @Override
- public String getPathForHadoop(String dataSource)
- {
- return getPathForHadoop();
- }
-
@Override
public DataSegment push(final File indexFilesDir, DataSegment segment, final boolean useUniquePath) throws IOException
{
diff --git a/extensions-contrib/cloudfiles-extensions/src/main/java/org/apache/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java b/extensions-contrib/cloudfiles-extensions/src/main/java/org/apache/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java
index 22ca44473b3..987530b2931 100644
--- a/extensions-contrib/cloudfiles-extensions/src/main/java/org/apache/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java
+++ b/extensions-contrib/cloudfiles-extensions/src/main/java/org/apache/druid/storage/cloudfiles/CloudFilesDataSegmentPusher.java
@@ -55,19 +55,6 @@ public class CloudFilesDataSegmentPusher implements DataSegmentPusher
this.jsonMapper = jsonMapper;
}
- @Override
- public String getPathForHadoop()
- {
- return null;
- }
-
- @Deprecated
- @Override
- public String getPathForHadoop(final String dataSource)
- {
- return getPathForHadoop();
- }
-
@Override
public DataSegment push(final File indexFilesDir, final DataSegment inSegment, final boolean useUniquePath)
{
diff --git a/extensions-contrib/materialized-view-maintenance/pom.xml b/extensions-contrib/materialized-view-maintenance/pom.xml
deleted file mode 100644
index 1bf750630d6..00000000000
--- a/extensions-contrib/materialized-view-maintenance/pom.xml
+++ /dev/null
@@ -1,142 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>druid</artifactId>
- <groupId>org.apache.druid</groupId>
- <version>37.0.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.druid.extensions.contrib</groupId>
- <artifactId>materialized-view-maintenance</artifactId>
- <name>materialized-view-maintenance</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-indexing-hadoop</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-indexing-service</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest-core</artifactId>
- <scope>test</scope>
- </dependency>
-</dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.owasp</groupId>
- <artifactId>dependency-check-maven</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadata.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadata.java
deleted file mode 100644
index 0c38b2fd3bd..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadata.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Sets;
-import org.apache.druid.indexing.overlord.DataSourceMetadata;
-
-import java.util.Objects;
-import java.util.Set;
-
-public class DerivativeDataSourceMetadata implements DataSourceMetadata
-{
- private final String baseDataSource;
- private final Set<String> dimensions;
- private final Set<String> metrics;
-
- @JsonCreator
- public DerivativeDataSourceMetadata(
- @JsonProperty("baseDataSource") String baseDataSource,
- @JsonProperty("dimensions") Set<String> dimensions,
- @JsonProperty("metrics") Set<String> metrics
- )
- {
- Preconditions.checkArgument(!Strings.isNullOrEmpty(baseDataSource), "baseDataSource cannot be null or empty. Please provide a baseDataSource.");
- this.baseDataSource = baseDataSource;
-
- this.dimensions = Preconditions.checkNotNull(dimensions, "dimensions cannot be null. This is not a valid DerivativeDataSourceMetadata.");
- this.metrics = Preconditions.checkNotNull(metrics, "metrics cannot be null. This is not a valid DerivativeDataSourceMetadata.");
- }
-
- @JsonProperty("baseDataSource")
- public String getBaseDataSource()
- {
- return baseDataSource;
- }
-
- @JsonProperty("dimensions")
- public Set<String> getDimensions()
- {
- return dimensions;
- }
-
- @JsonProperty("metrics")
- public Set<String> getMetrics()
- {
- return metrics;
- }
-
- @Override
- public boolean isValidStart()
- {
- return false;
- }
-
- @Override
- public DataSourceMetadata asStartMetadata()
- {
- return this;
- }
-
- @Override
- public boolean matches(DataSourceMetadata other)
- {
- return equals(other);
- }
-
- @Override
- public DataSourceMetadata plus(DataSourceMetadata other)
- {
- throw new UnsupportedOperationException("Derivative dataSource metadata is not allowed to plus");
- }
-
- @Override
- public DataSourceMetadata minus(DataSourceMetadata other)
- {
- throw new UnsupportedOperationException("Derivative dataSource metadata is not allowed to minus");
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DerivativeDataSourceMetadata that = (DerivativeDataSourceMetadata) o;
-
- return baseDataSource.equals(that.getBaseDataSource()) &&
- dimensions.equals(that.getDimensions()) &&
- metrics.equals(that.getMetrics());
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(baseDataSource, dimensions, metrics);
- }
-
- public Set<String> getColumns()
- {
- Set<String> fields = Sets.newHashSet(dimensions);
- fields.addAll(metrics);
- return fields;
- }
-
- @Override
- public String toString()
- {
- return "DerivedDataSourceMetadata{" +
- "baseDataSource=" + baseDataSource +
- ", dimensions=" + dimensions +
- ", metrics=" + metrics +
- '}';
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewMaintenanceDruidModule.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewMaintenanceDruidModule.java
deleted file mode 100644
index 795dd54a034..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewMaintenanceDruidModule.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.common.collect.ImmutableList;
-import com.google.inject.Binder;
-import org.apache.druid.discovery.NodeRole;
-import org.apache.druid.guice.JsonConfigProvider;
-import org.apache.druid.guice.annotations.LoadScope;
-import org.apache.druid.initialization.DruidModule;
-
-import java.util.List;
-
-@LoadScope(roles = NodeRole.OVERLORD_JSON_NAME)
-public class MaterializedViewMaintenanceDruidModule implements DruidModule
-{
- @Override
- public List<? extends Module> getJacksonModules()
- {
- return ImmutableList.of(
- new SimpleModule(getClass().getSimpleName())
- .registerSubtypes(
- new NamedType(MaterializedViewSupervisorSpec.class, "derivativeDataSource"),
- new NamedType(DerivativeDataSourceMetadata.class, "derivativeDataSource")
- )
- );
- }
-
- @Override
- public void configure(Binder binder)
- {
- JsonConfigProvider.bind(binder, "druid.materialized.view.task", MaterializedViewTaskConfig.class);
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
deleted file mode 100644
index c3bd7bb41bc..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisor.java
+++ /dev/null
@@ -1,517 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.MapDifference;
-import com.google.common.collect.Maps;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import org.apache.druid.error.DruidException;
-import org.apache.druid.error.EntryAlreadyExists;
-import org.apache.druid.indexer.HadoopIndexTask;
-import org.apache.druid.indexer.TaskStatus;
-import org.apache.druid.indexing.overlord.DataSourceMetadata;
-import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
-import org.apache.druid.indexing.overlord.Segments;
-import org.apache.druid.indexing.overlord.TaskMaster;
-import org.apache.druid.indexing.overlord.TaskStorage;
-import org.apache.druid.indexing.overlord.supervisor.Supervisor;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorReport;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager;
-import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.JodaUtils;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.concurrent.Execs;
-import org.apache.druid.java.util.common.guava.Comparators;
-import org.apache.druid.java.util.emitter.EmittingLogger;
-import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.timeline.DataSegment;
-import org.joda.time.Duration;
-import org.joda.time.Interval;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-import java.util.TreeMap;
-import java.util.concurrent.TimeUnit;
-
-public class MaterializedViewSupervisor implements Supervisor
-{
- private static final EmittingLogger log = new EmittingLogger(MaterializedViewSupervisor.class);
- private static final int DEFAULT_MAX_TASK_COUNT = 1;
- // there is a lag between derivatives and base dataSource, to prevent repeatedly building for some delay data.
- private static final long DEFAULT_MIN_DATA_LAG_MS = TimeUnit.DAYS.toMillis(1);
-
- private final MetadataSupervisorManager metadataSupervisorManager;
- private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private final MaterializedViewSupervisorSpec spec;
- private final TaskMaster taskMaster;
- private final TaskStorage taskStorage;
- private final MaterializedViewTaskConfig config;
- private final SupervisorStateManager stateManager;
- private final String dataSource;
- private final String supervisorId;
- private final int maxTaskCount;
- private final long minDataLagMs;
- private final Map<Interval, HadoopIndexTask> runningTasks = new HashMap<>();
- private final Map<Interval, String> runningVersion = new HashMap<>();
- // taskLock is used to synchronize runningTask and runningVersion
- private final Object taskLock = new Object();
- // stateLock is used to synchronize materializedViewSupervisor's status
- private final Object stateLock = new Object();
- private boolean started = false;
- private ListenableFuture<?> future = null;
- private ListeningScheduledExecutorService exec = null;
- // In the missing intervals, baseDataSource has data but derivedDataSource does not, which means
- // data in these intervals of derivedDataSource needs to be rebuilt.
- private Set<Interval> missInterval = new HashSet<>();
-
- public MaterializedViewSupervisor(
- TaskMaster taskMaster,
- TaskStorage taskStorage,
- MetadataSupervisorManager metadataSupervisorManager,
- IndexerMetadataStorageCoordinator metadataStorageCoordinator,
- MaterializedViewTaskConfig config,
- MaterializedViewSupervisorSpec spec
- )
- {
- this.taskMaster = taskMaster;
- this.taskStorage = taskStorage;
- this.metadataStorageCoordinator = metadataStorageCoordinator;
- this.metadataSupervisorManager = metadataSupervisorManager;
- this.config = config;
- this.spec = spec;
- this.stateManager = new SupervisorStateManager(spec.getSupervisorStateManagerConfig(), spec.isSuspended());
- this.dataSource = spec.getDataSourceName();
- this.supervisorId = StringUtils.format("MaterializedViewSupervisor-%s", dataSource);
- this.maxTaskCount = spec.getContext().containsKey("maxTaskCount")
- ? Integer.parseInt(String.valueOf(spec.getContext().get("maxTaskCount")))
- : DEFAULT_MAX_TASK_COUNT;
- this.minDataLagMs = spec.getContext().containsKey("minDataLagMs")
- ? Long.parseLong(String.valueOf(spec.getContext().get("minDataLagMs")))
- : DEFAULT_MIN_DATA_LAG_MS;
- }
-
- @Override
- public void start()
- {
- synchronized (stateLock) {
- Preconditions.checkState(!started, "already started");
-
- DataSourceMetadata metadata = metadataStorageCoordinator.retrieveDataSourceMetadata(dataSource);
- if (null == metadata) {
- metadataStorageCoordinator.insertDataSourceMetadata(
- dataSource,
- new DerivativeDataSourceMetadata(spec.getBaseDataSource(), spec.getDimensions(), spec.getMetrics())
- );
- }
- exec = MoreExecutors.listeningDecorator(Execs.scheduledSingleThreaded(StringUtils.encodeForFormat(supervisorId)));
- final Duration delay = config.getTaskCheckDuration().toStandardDuration();
- future = exec.scheduleWithFixedDelay(
- MaterializedViewSupervisor.this::run,
- 0,
- delay.getMillis(),
- TimeUnit.MILLISECONDS
- );
- started = true;
- }
- }
-
- @VisibleForTesting
- public void run()
- {
- try {
- if (spec.isSuspended()) {
- log.info(
- "Materialized view supervisor[%s:%s] is suspended",
- spec.getId(),
- spec.getDataSourceName()
- );
- return;
- }
-
- DataSourceMetadata metadata = metadataStorageCoordinator.retrieveDataSourceMetadata(dataSource);
- if (metadata instanceof DerivativeDataSourceMetadata
- && spec.getBaseDataSource().equals(((DerivativeDataSourceMetadata) metadata).getBaseDataSource())
- && spec.getDimensions().equals(((DerivativeDataSourceMetadata) metadata).getDimensions())
- && spec.getMetrics().equals(((DerivativeDataSourceMetadata) metadata).getMetrics())) {
- checkSegmentsAndSubmitTasks();
- } else {
- log.error(
- "Failed to start %s. Metadata in database(%s) is different from new dataSource metadata(%s)",
- supervisorId,
- metadata,
- spec
- );
- }
- }
- catch (Exception e) {
- stateManager.recordThrowableEvent(e);
- log.makeAlert(e, StringUtils.format("uncaught exception in %s.", supervisorId)).emit();
- }
- finally {
- stateManager.markRunFinished();
- }
- }
-
- @Override
- public void stop(boolean stopGracefully)
- {
- synchronized (stateLock) {
- Preconditions.checkState(started, "not started");
-
- stateManager.maybeSetState(SupervisorStateManager.BasicState.STOPPING);
-
- // stop all schedulers and threads
- if (stopGracefully) {
- synchronized (taskLock) {
- future.cancel(false);
- future = null;
- exec.shutdownNow();
- exec = null;
- clearTasks();
- if (!(metadataSupervisorManager.getLatest().get(supervisorId) instanceof MaterializedViewSupervisorSpec)) {
- clearSegments();
- }
- }
- } else {
- future.cancel(true);
- future = null;
- exec.shutdownNow();
- exec = null;
- synchronized (taskLock) {
- clearTasks();
- if (!(metadataSupervisorManager.getLatest().get(supervisorId) instanceof MaterializedViewSupervisorSpec)) {
- clearSegments();
- }
- }
- }
- started = false;
- }
-
- }
-
- @Override
- public SupervisorReport getStatus()
- {
- return new MaterializedViewSupervisorReport(
- dataSource,
- DateTimes.nowUtc(),
- spec.isSuspended(),
- spec.getBaseDataSource(),
- spec.getDimensions(),
- spec.getMetrics(),
- JodaUtils.condenseIntervals(missInterval),
- stateManager.isHealthy(),
- stateManager.getSupervisorState().getBasicState(),
- stateManager.getExceptionEvents()
- );
- }
-
- @Override
- public SupervisorStateManager.State getState()
- {
- return stateManager.getSupervisorState();
- }
-
- @Override
- public Boolean isHealthy()
- {
- return stateManager.isHealthy();
- }
-
- @Override
- public void reset(@Nullable DataSourceMetadata dataSourceMetadata)
- {
- if (dataSourceMetadata == null) {
- // if oldMetadata is different from spec, tasks and segments will be removed when reset.
- DataSourceMetadata oldMetadata = metadataStorageCoordinator.retrieveDataSourceMetadata(dataSource);
- if (oldMetadata instanceof DerivativeDataSourceMetadata) {
- if (!((DerivativeDataSourceMetadata) oldMetadata).getBaseDataSource().equals(spec.getBaseDataSource()) ||
- !((DerivativeDataSourceMetadata) oldMetadata).getDimensions().equals(spec.getDimensions()) ||
- !((DerivativeDataSourceMetadata) oldMetadata).getMetrics().equals(spec.getMetrics())) {
- synchronized (taskLock) {
- clearTasks();
- clearSegments();
- }
- }
- }
- commitDataSourceMetadata(
- new DerivativeDataSourceMetadata(spec.getBaseDataSource(), spec.getDimensions(), spec.getMetrics())
- );
- } else {
- throw new IAE("DerivedDataSourceMetadata is not allowed to reset to a new DerivedDataSourceMetadata");
- }
- }
-
- /**
- * Find intervals in which derived dataSource should rebuild the segments.
- * Choose the latest intervals to create new HadoopIndexTask and submit it.
- */
- @VisibleForTesting
- void checkSegmentsAndSubmitTasks()
- {
- synchronized (taskLock) {
- List<Interval> intervalsToRemove = new ArrayList<>();
- for (Map.Entry<Interval, HadoopIndexTask> entry : runningTasks.entrySet()) {
- Optional<TaskStatus> taskStatus = taskStorage.getStatus(entry.getValue().getId());
- if (!taskStatus.isPresent() || !taskStatus.get().isRunnable()) {
- intervalsToRemove.add(entry.getKey());
- }
- }
- for (Interval interval : intervalsToRemove) {
- runningTasks.remove(interval);
- runningVersion.remove(interval);
- }
-
- if (runningTasks.size() == maxTaskCount) {
- //if the number of running tasks reach the max task count, supervisor won't submit new tasks.
- return;
- }
- Pair<SortedMap<Interval, String>, Map<Interval, List<DataSegment>>> toBuildIntervalAndBaseSegments =
- checkSegments();
- SortedMap<Interval, String> sortedToBuildVersion = toBuildIntervalAndBaseSegments.lhs;
- Map<Interval, List<DataSegment>> baseSegments = toBuildIntervalAndBaseSegments.rhs;
- missInterval = sortedToBuildVersion.keySet();
- submitTasks(sortedToBuildVersion, baseSegments);
- }
- }
-
- @VisibleForTesting
- Pair<Map<Interval, HadoopIndexTask>, Map<Interval, String>> getRunningTasks()
- {
- return new Pair<>(runningTasks, runningVersion);
- }
-
- /**
- * Find infomation about the intervals in which derived dataSource data should be rebuilt.
- * The infomation includes the version and DataSegments list of a interval.
- * The intervals include: in the interval,
- * 1) baseDataSource has data, but the derivedDataSource does not;
- * 2) version of derived segments isn't the max(created_date) of all base segments;
- *
- * Drop the segments of the intervals in which derivedDataSource has data, but baseDataSource does not.
- *
- * @return the left part of Pair: interval -> version, and the right part: interval -> DataSegment list.
- * Version and DataSegment list can be used to create HadoopIndexTask.
- * Derived datasource data in all these intervals need to be rebuilt.
- */
- @VisibleForTesting
- Pair<SortedMap<Interval, String>, Map<Interval, List<DataSegment>>> checkSegments()
- {
- // Pair<interval -> version, interval -> list<DataSegment>>
- Collection<DataSegment> derivativeSegmentsCollection =
- metadataStorageCoordinator.retrieveAllUsedSegments(dataSource, Segments.ONLY_VISIBLE);
- Pair<Map<Interval, String>, Map<Interval, List<DataSegment>>> derivativeSegmentsSnapshot =
- getVersionAndBaseSegments(derivativeSegmentsCollection);
- // Pair<interval -> max(created_date), interval -> list<DataSegment>>
- Pair<Map<Interval, String>, Map<Interval, List<DataSegment>>> baseSegmentsSnapshot =
- getMaxCreateDateAndBaseSegments(
- metadataStorageCoordinator.retrieveUsedSegmentsAndCreatedDates(spec.getBaseDataSource(),
- Collections.singletonList(Intervals.ETERNITY))
- );
- // baseSegments are used to create HadoopIndexTask
- Map<Interval, List<DataSegment>> baseSegments = baseSegmentsSnapshot.rhs;
- Map<Interval, List<DataSegment>> derivativeSegments = derivativeSegmentsSnapshot.rhs;
- // use max created_date of base segments as the version of derivative segments
- Map<Interval, String> maxCreatedDate = baseSegmentsSnapshot.lhs;
- Map<Interval, String> derivativeVersion = derivativeSegmentsSnapshot.lhs;
- SortedMap<Interval, String> sortedToBuildInterval =
- new TreeMap<>(Comparators.intervalsByStartThenEnd().reversed());
- // find the intervals to drop and to build
- MapDifference<Interval, String> difference = Maps.difference(maxCreatedDate, derivativeVersion);
- Map<Interval, String> toBuildInterval = new HashMap<>(difference.entriesOnlyOnLeft());
- Map<Interval, String> toDropInterval = new HashMap<>(difference.entriesOnlyOnRight());
- // update version of derived segments if isn't the max (created_date) of all base segments
- // prevent user supplied segments list did not match with segments list obtained from db
- Map<Interval, MapDifference.ValueDifference<String>> checkIfNewestVersion =
- new HashMap<>(difference.entriesDiffering());
- for (Map.Entry<Interval, MapDifference.ValueDifference<String>> entry : checkIfNewestVersion.entrySet()) {
- final String versionOfBase = maxCreatedDate.get(entry.getKey());
- final String versionOfDerivative = derivativeVersion.get(entry.getKey());
- final int baseCount = baseSegments.get(entry.getKey()).size();
- if (versionOfBase.compareTo(versionOfDerivative) > 0) {
- int usedCount = metadataStorageCoordinator
- .retrieveUsedSegmentsForInterval(spec.getBaseDataSource(), entry.getKey(), Segments.ONLY_VISIBLE).size();
- if (baseCount == usedCount) {
- toBuildInterval.put(entry.getKey(), versionOfBase);
- }
- }
- }
- // if some intervals are in running tasks and the versions are the same, remove it from toBuildInterval
- // if some intervals are in running tasks, but the versions are different, stop the task.
- runningVersion.forEach((interval, version) -> {
- if (toBuildInterval.containsKey(interval)) {
- if (toBuildInterval.get(interval).equals(version)) {
- toBuildInterval.remove(interval);
- } else {
- if (taskMaster.getTaskQueue().isPresent()) {
- taskMaster.getTaskQueue().get().shutdown(runningTasks.get(interval).getId(), "version mismatch");
- runningTasks.remove(interval);
- }
- }
- }
- });
- // drop derivative segments which interval equals the interval in toDeleteBaseSegments
- for (Interval interval : toDropInterval.keySet()) {
- for (DataSegment segment : derivativeSegments.get(interval)) {
- metadataStorageCoordinator.markSegmentAsUnused(segment.getId());
- }
- }
- // data of the latest interval will be built firstly.
- sortedToBuildInterval.putAll(toBuildInterval);
- return new Pair<>(sortedToBuildInterval, baseSegments);
- }
-
- private void submitTasks(
- SortedMap<Interval, String> sortedToBuildVersion,
- Map<Interval, List<DataSegment>> baseSegments
- )
- {
- for (Map.Entry<Interval, String> entry : sortedToBuildVersion.entrySet()) {
- if (runningTasks.size() < maxTaskCount) {
- HadoopIndexTask task = spec.createTask(entry.getKey(), entry.getValue(), baseSegments.get(entry.getKey()));
- try {
- if (taskMaster.getTaskQueue().isPresent()) {
- taskMaster.getTaskQueue().get().add(task);
- runningVersion.put(entry.getKey(), entry.getValue());
- runningTasks.put(entry.getKey(), task);
- }
- }
- catch (DruidException e) {
- if (EntryAlreadyExists.ERROR_CODE.equals(e.getErrorCode())) {
- log.error("Task[%s] already exists", task.getId());
- } else {
- throw e;
- }
- }
- catch (RuntimeException e) {
- throw e;
- }
- catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-
- private Pair<Map<Interval, String>, Map<Interval, List<DataSegment>>> getVersionAndBaseSegments(
- Collection<DataSegment> snapshot
- )
- {
- Map<Interval, String> versions = new HashMap<>();
- Map<Interval, List<DataSegment>> segments = new HashMap<>();
- for (DataSegment segment : snapshot) {
- Interval interval = segment.getInterval();
- versions.put(interval, segment.getVersion());
- segments.computeIfAbsent(interval, i -> new ArrayList<>()).add(segment);
- }
- return new Pair<>(versions, segments);
- }
-
- private Pair<Map<Interval, String>, Map<Interval, List<DataSegment>>> getMaxCreateDateAndBaseSegments(
- Collection<Pair<DataSegment, String>> snapshot
- )
- {
- Interval maxAllowedToBuildInterval = snapshot.parallelStream()
- .map(pair -> pair.lhs)
- .map(DataSegment::getInterval)
- .max(Comparators.intervalsByStartThenEnd())
- .get();
- Map<Interval, String> maxCreatedDate = new HashMap<>();
- Map<Interval, List<DataSegment>> segments = new HashMap<>();
- for (Pair<DataSegment, String> entry : snapshot) {
- DataSegment segment = entry.lhs;
- String createDate = entry.rhs;
- Interval interval = segment.getInterval();
- if (!hasEnoughLag(interval, maxAllowedToBuildInterval)) {
- continue;
- }
- maxCreatedDate.merge(interval, createDate, (date1, date2) -> {
- return DateTimes.max(DateTimes.of(date1), DateTimes.of(date2)).toString();
- });
- segments.computeIfAbsent(interval, i -> new ArrayList<>()).add(segment);
- }
- return new Pair<>(maxCreatedDate, segments);
- }
-
-
- /**
- * check whether the start millis of target interval is more than minDataLagMs lagging behind maxInterval's
- * minDataLag is required to prevent repeatedly building data because of delay data.
- *
- * @param target
- * @param maxInterval
- * @return true if the start millis of target interval is more than minDataLagMs lagging behind maxInterval's
- */
- private boolean hasEnoughLag(Interval target, Interval maxInterval)
- {
- return minDataLagMs <= (maxInterval.getStartMillis() - target.getStartMillis());
- }
-
- private void clearTasks()
- {
- for (HadoopIndexTask task : runningTasks.values()) {
- if (taskMaster.getTaskQueue().isPresent()) {
- taskMaster.getTaskQueue().get().shutdown(task.getId(), "killing all tasks");
- }
- }
- runningTasks.clear();
- runningVersion.clear();
- }
-
- private void clearSegments()
- {
- log.info("Clear all metadata of dataSource %s", dataSource);
- metadataStorageCoordinator.deletePendingSegments(dataSource);
- metadataStorageCoordinator.markAllSegmentsAsUnused(dataSource);
- metadataStorageCoordinator.deleteDataSourceMetadata(dataSource);
- }
-
- private void commitDataSourceMetadata(DataSourceMetadata dataSourceMetadata)
- {
- if (!metadataStorageCoordinator.insertDataSourceMetadata(dataSource, dataSourceMetadata)) {
- try {
- metadataStorageCoordinator.resetDataSourceMetadata(
- dataSource,
- dataSourceMetadata
- );
- }
- catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorReport.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorReport.java
deleted file mode 100644
index 13e51dafd14..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorReport;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManager;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
-
-import java.util.List;
-import java.util.Set;
-
-public class MaterializedViewSupervisorReport extends SupervisorReport
-{
- public MaterializedViewSupervisorReport(
- String dataSource,
- DateTime generationTime,
- boolean suspended,
- String baseDataSource,
- Set<String> dimensions,
- Set<String> metrics,
- List<Interval> missTimeline,
- boolean healthy,
- SupervisorStateManager.State state,
- List<SupervisorStateManager.ExceptionEvent> recentErrors
- )
- {
- super(
- dataSource,
- generationTime,
- ImmutableMap.builder()
- .put("dataSource", dataSource)
- .put("baseDataSource", baseDataSource)
- .put("suspended", suspended)
- .put("dimensions", dimensions)
- .put("metrics", metrics)
- .put("missTimeline", Sets.newHashSet(missTimeline))
- .put("healthy", healthy)
- .put("state", state)
- .put("recentErrors", recentErrors)
- .build()
- );
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
deleted file mode 100644
index c835103c4c2..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpec.java
+++ /dev/null
@@ -1,432 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.ImmutableList;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.indexer.HadoopIOConfig;
-import org.apache.druid.indexer.HadoopIndexTask;
-import org.apache.druid.indexer.HadoopIngestionSpec;
-import org.apache.druid.indexer.HadoopTaskConfig;
-import org.apache.druid.indexer.HadoopTuningConfig;
-import org.apache.druid.indexer.granularity.ArbitraryGranularitySpec;
-import org.apache.druid.indexer.hadoop.DatasourceIngestionSpec;
-import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
-import org.apache.druid.indexing.overlord.TaskMaster;
-import org.apache.druid.indexing.overlord.TaskStorage;
-import org.apache.druid.indexing.overlord.supervisor.Supervisor;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorSpec;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig;
-import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.granularity.Granularities;
-import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.realtime.ChatHandlerProvider;
-import org.apache.druid.server.security.AuthorizerMapper;
-import org.apache.druid.timeline.DataSegment;
-import org.joda.time.Interval;
-
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class MaterializedViewSupervisorSpec implements SupervisorSpec
-{
- private static final String TASK_PREFIX = "index_materialized_view";
- private static final String SUPERVISOR_TYPE = "materialized_view";
- private final String baseDataSource;
- private final DimensionsSpec dimensionsSpec;
- private final AggregatorFactory[] aggregators;
- private final HadoopTuningConfig tuningConfig;
- private final String dataSourceName;
- private final String hadoopCoordinates;
- private final List<String> hadoopDependencyCoordinates;
- private final String classpathPrefix;
- private final Map<String, Object> context;
- private final Set<String> metrics;
- private final Set<String> dimensions;
- private final ObjectMapper objectMapper;
- private final MetadataSupervisorManager metadataSupervisorManager;
- private final IndexerMetadataStorageCoordinator metadataStorageCoordinator;
- private final TaskMaster taskMaster;
- private final TaskStorage taskStorage;
- private final MaterializedViewTaskConfig config;
- private final AuthorizerMapper authorizerMapper;
- private final ChatHandlerProvider chatHandlerProvider;
- private final SupervisorStateManagerConfig supervisorStateManagerConfig;
- private final boolean suspended;
-
- public MaterializedViewSupervisorSpec(
- @JsonProperty("baseDataSource") String baseDataSource,
- @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec,
- @JsonProperty("metricsSpec") AggregatorFactory[] aggregators,
- @JsonProperty("tuningConfig") HadoopTuningConfig tuningConfig,
- @JsonProperty("dataSource") String dataSourceName,
- @JsonProperty("hadoopCoordinates") String hadoopCoordinates,
- @JsonProperty("hadoopDependencyCoordinates") List<String> hadoopDependencyCoordinates,
- @JsonProperty("classpathPrefix") String classpathPrefix,
- @JsonProperty("context") Map<String, Object> context,
- @JsonProperty("suspended") Boolean suspended,
- @JacksonInject ObjectMapper objectMapper,
- @JacksonInject TaskMaster taskMaster,
- @JacksonInject TaskStorage taskStorage,
- @JacksonInject MetadataSupervisorManager metadataSupervisorManager,
- @JacksonInject IndexerMetadataStorageCoordinator metadataStorageCoordinator,
- @JacksonInject MaterializedViewTaskConfig config,
- @JacksonInject AuthorizerMapper authorizerMapper,
- @JacksonInject ChatHandlerProvider chatHandlerProvider,
- @JacksonInject SupervisorStateManagerConfig supervisorStateManagerConfig
- )
- {
- Preconditions.checkArgument(
- !Strings.isNullOrEmpty(baseDataSource),
- "baseDataSource cannot be null or empty. Please provide a baseDataSource."
- );
- this.baseDataSource = baseDataSource;
-
- this.dimensionsSpec = Preconditions.checkNotNull(
- dimensionsSpec,
- "dimensionsSpec cannot be null. Please provide a dimensionsSpec"
- );
- this.aggregators = Preconditions.checkNotNull(
- aggregators,
- "metricsSpec cannot be null. Please provide a metricsSpec"
- );
- this.tuningConfig = Preconditions.checkNotNull(
- tuningConfig,
- "tuningConfig cannot be null. Please provide tuningConfig"
- );
-
- this.dataSourceName = dataSourceName == null
- ? StringUtils.format(
- "%s-%s",
- baseDataSource,
- DigestUtils.sha1Hex(dimensionsSpec.toString()).substring(0, 8)
- )
- : dataSourceName;
- this.hadoopCoordinates = hadoopCoordinates;
- this.hadoopDependencyCoordinates = hadoopDependencyCoordinates;
- this.classpathPrefix = classpathPrefix;
- this.context = context == null ? new HashMap<>() : context;
- this.objectMapper = objectMapper;
- this.taskMaster = taskMaster;
- this.taskStorage = taskStorage;
- this.metadataSupervisorManager = metadataSupervisorManager;
- this.metadataStorageCoordinator = metadataStorageCoordinator;
- this.authorizerMapper = authorizerMapper;
- this.chatHandlerProvider = chatHandlerProvider;
- this.config = config;
- this.supervisorStateManagerConfig = supervisorStateManagerConfig;
- this.suspended = suspended != null ? suspended : false;
-
- this.metrics = new HashSet<>();
- for (AggregatorFactory aggregatorFactory : aggregators) {
- metrics.add(aggregatorFactory.getName());
- }
- this.dimensions = new HashSet<>();
- for (DimensionSchema schema : dimensionsSpec.getDimensions()) {
- dimensions.add(schema.getName());
- }
- }
-
- public HadoopIndexTask createTask(Interval interval, String version, List<DataSegment> segments)
- {
- String taskId = StringUtils.format("%s_%s_%s", TASK_PREFIX, dataSourceName, DateTimes.nowUtc());
-
- // generate parser
- Map<String, Object> parseSpec = new HashMap<>();
- parseSpec.put("format", "timeAndDims");
- parseSpec.put("dimensionsSpec", dimensionsSpec);
- Map<String, Object> parser = new HashMap<>();
- parser.put("type", "map");
- parser.put("parseSpec", parseSpec);
-
- //generate HadoopTuningConfig
- HadoopTuningConfig tuningConfigForTask = new HadoopTuningConfig(
- tuningConfig.getWorkingPath(),
- version,
- tuningConfig.getPartitionsSpec(),
- tuningConfig.getShardSpecs(),
- tuningConfig.getIndexSpec(),
- tuningConfig.getIndexSpecForIntermediatePersists(),
- tuningConfig.getAppendableIndexSpec(),
- tuningConfig.getMaxRowsInMemory(),
- tuningConfig.getMaxBytesInMemory(),
- tuningConfig.isLeaveIntermediate(),
- tuningConfig.isCleanupOnFailure(),
- tuningConfig.isOverwriteFiles(),
- tuningConfig.isIgnoreInvalidRows(),
- tuningConfig.getJobProperties(),
- tuningConfig.isCombineText(),
- tuningConfig.getUseCombiner(),
- tuningConfig.getMaxRowsInMemory(),
- tuningConfig.getNumBackgroundPersistThreads(),
- tuningConfig.isForceExtendableShardSpecs(),
- true,
- tuningConfig.getUserAllowedHadoopPrefix(),
- tuningConfig.isLogParseExceptions(),
- tuningConfig.getMaxParseExceptions(),
- tuningConfig.isUseYarnRMJobStatusFallback(),
- tuningConfig.getAwaitSegmentAvailabilityTimeoutMillis(),
- HadoopTuningConfig.DEFAULT_DETERMINE_PARTITIONS_SAMPLING_FACTOR
- );
-
- // generate granularity
- ArbitraryGranularitySpec granularitySpec = new ArbitraryGranularitySpec(
- Granularities.NONE,
- ImmutableList.of(interval)
- );
-
- // generate DataSchema
- DataSchema dataSchema = DataSchema.builder()
- .withDataSource(dataSourceName)
- .withParserMap(parser)
- .withAggregators(aggregators)
- .withGranularity(granularitySpec)
- .withObjectMapper(objectMapper)
- .build();
-
- // generate DatasourceIngestionSpec
- DatasourceIngestionSpec datasourceIngestionSpec = new DatasourceIngestionSpec(
- baseDataSource,
- null,
- ImmutableList.of(interval),
- segments,
- null,
- null,
- null,
- false,
- null
- );
-
- // generate HadoopIOConfig
- Map<String, Object> inputSpec = new HashMap<>();
- inputSpec.put("type", "dataSource");
- inputSpec.put("ingestionSpec", datasourceIngestionSpec);
- HadoopIOConfig hadoopIOConfig = new HadoopIOConfig(inputSpec, null, null);
-
- // generate HadoopIngestionSpec
- HadoopIngestionSpec spec = new HadoopIngestionSpec(dataSchema, hadoopIOConfig, tuningConfigForTask);
-
- // generate HadoopIndexTask
- HadoopIndexTask task = new HadoopIndexTask(
- taskId,
- spec,
- hadoopCoordinates,
- hadoopDependencyCoordinates,
- classpathPrefix,
- objectMapper,
- context,
- authorizerMapper,
- chatHandlerProvider,
- new HadoopTaskConfig(null, null)
- );
-
- return task;
- }
-
- public Set<String> getDimensions()
- {
- return dimensions;
- }
-
- public Set<String> getMetrics()
- {
- return metrics;
- }
-
- @JsonProperty("baseDataSource")
- public String getBaseDataSource()
- {
- return baseDataSource;
- }
-
- @JsonProperty("dimensionsSpec")
- public DimensionsSpec getDimensionsSpec()
- {
- return dimensionsSpec;
- }
-
- @JsonProperty("metricsSpec")
- public AggregatorFactory[] getMetricsSpec()
- {
- return aggregators;
- }
-
- @JsonProperty("tuningConfig")
- public HadoopTuningConfig getTuningConfig()
- {
- return tuningConfig;
- }
-
- @JsonProperty("dataSource")
- public String getDataSourceName()
- {
- return dataSourceName;
- }
-
- @JsonProperty("hadoopCoordinates")
- public String getHadoopCoordinates()
- {
- return hadoopCoordinates;
- }
-
- @JsonProperty("hadoopDependencyCoordinates")
- public List<String> getSadoopDependencyCoordinates()
- {
- return hadoopDependencyCoordinates;
- }
-
- @JsonProperty("classpathPrefix")
- public String getClasspathPrefix()
- {
- return classpathPrefix;
- }
-
- @JsonProperty("context")
- public Map<String, Object> getContext()
- {
- return context;
- }
-
- @Override
- @JsonProperty("suspended")
- public boolean isSuspended()
- {
- return suspended;
- }
-
- @Override
- @JsonProperty("type")
- public String getType()
- {
- return SUPERVISOR_TYPE;
- }
-
- @Override
- @JsonProperty("source")
- public String getSource()
- {
- return getBaseDataSource();
- }
-
- @Override
- public String getId()
- {
- return StringUtils.format("MaterializedViewSupervisor-%s", dataSourceName);
- }
-
- @Override
- public Supervisor createSupervisor()
- {
- return new MaterializedViewSupervisor(
- taskMaster,
- taskStorage,
- metadataSupervisorManager,
- metadataStorageCoordinator,
- config,
- this
- );
- }
-
- @Override
- public List<String> getDataSources()
- {
- return ImmutableList.of(dataSourceName);
- }
-
- @Override
- public SupervisorSpec createSuspendedSpec()
- {
- return new MaterializedViewSupervisorSpec(
- baseDataSource,
- dimensionsSpec,
- aggregators,
- tuningConfig,
- dataSourceName,
- hadoopCoordinates,
- hadoopDependencyCoordinates,
- classpathPrefix,
- context,
- true,
- objectMapper,
- taskMaster,
- taskStorage,
- metadataSupervisorManager,
- metadataStorageCoordinator,
- config,
- authorizerMapper,
- chatHandlerProvider,
- supervisorStateManagerConfig
- );
- }
-
- @Override
- public SupervisorSpec createRunningSpec()
- {
- return new MaterializedViewSupervisorSpec(
- baseDataSource,
- dimensionsSpec,
- aggregators,
- tuningConfig,
- dataSourceName,
- hadoopCoordinates,
- hadoopDependencyCoordinates,
- classpathPrefix,
- context,
- false,
- objectMapper,
- taskMaster,
- taskStorage,
- metadataSupervisorManager,
- metadataStorageCoordinator,
- config,
- authorizerMapper,
- chatHandlerProvider,
- supervisorStateManagerConfig
- );
- }
-
- public SupervisorStateManagerConfig getSupervisorStateManagerConfig()
- {
- return supervisorStateManagerConfig;
- }
-
- @Override
- public String toString()
- {
- return "MaterializedViewSupervisorSpec{" +
- "baseDataSource=" + baseDataSource +
- ", dimensions=" + dimensions +
- ", metrics=" + metrics +
- '}';
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewTaskConfig.java b/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewTaskConfig.java
deleted file mode 100644
index 1a63420523d..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/java/org/apache/druid/indexing/materializedview/MaterializedViewTaskConfig.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.joda.time.Period;
-
-public class MaterializedViewTaskConfig
-{
- @JsonProperty
- private Period taskCheckDuration = new Period("PT1M");
-
- public Period getTaskCheckDuration()
- {
- return taskCheckDuration;
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/materialized-view-maintenance/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
deleted file mode 100644
index 935023d68b4..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.druid.indexing.materializedview.MaterializedViewMaintenanceDruidModule
diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadataTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadataTest.java
deleted file mode 100644
index 64a88f98fc3..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/DerivativeDataSourceMetadataTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.google.common.collect.Sets;
-import org.hamcrest.CoreMatchers;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.Set;
-
-
-public class DerivativeDataSourceMetadataTest
-{
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @Test
- public void testEmptyBaseDataSource()
- {
- expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
- expectedException.expectMessage(
- "baseDataSource cannot be null or empty. Please provide a baseDataSource."
- );
- String baseDataSource = "";
- Set<String> dims = Sets.newHashSet("dim1", "dim2", "dim3");
- Set<String> metrics = Sets.newHashSet("cost");
- DerivativeDataSourceMetadata metadata = new DerivativeDataSourceMetadata(baseDataSource, dims, metrics);
- }
-
- @Test
- public void testNullBaseDataSource()
- {
- expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
- expectedException.expectMessage(
- "baseDataSource cannot be null or empty. Please provide a baseDataSource."
- );
- String baseDataSource = null;
- Set<String> dims = Sets.newHashSet("dim1", "dim2", "dim3");
- Set<String> metrics = Sets.newHashSet("cost");
- DerivativeDataSourceMetadata metadata = new DerivativeDataSourceMetadata(baseDataSource, dims, metrics);
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java
deleted file mode 100644
index d690e007640..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorSpecTest.java
+++ /dev/null
@@ -1,346 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.databind.InjectableValues;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.google.common.collect.Lists;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.StringDimensionSchema;
-import org.apache.druid.indexer.HadoopTuningConfig;
-import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
-import org.apache.druid.indexing.overlord.TaskMaster;
-import org.apache.druid.indexing.overlord.TaskStorage;
-import org.apache.druid.indexing.overlord.supervisor.Supervisor;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig;
-import org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler;
-import org.apache.druid.math.expr.ExprMacroTable;
-import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.CountAggregatorFactory;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
-import org.apache.druid.segment.TestHelper;
-import org.apache.druid.segment.realtime.ChatHandlerProvider;
-import org.apache.druid.segment.realtime.NoopChatHandlerProvider;
-import org.apache.druid.server.security.AuthorizerMapper;
-import org.easymock.EasyMock;
-import org.hamcrest.CoreMatchers;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.io.IOException;
-
-public class MaterializedViewSupervisorSpecTest
-{
- @Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- private final ObjectMapper objectMapper = TestHelper.makeJsonMapper();
-
- @Before
- public void setup()
- {
- objectMapper.registerSubtypes(new NamedType(MaterializedViewSupervisorSpec.class, "derivativeDataSource"));
- objectMapper.setInjectableValues(
- new InjectableValues.Std()
- .addValue(TaskMaster.class, null)
- .addValue(TaskStorage.class, null)
- .addValue(ExprMacroTable.class.getName(), LookupEnabledTestExprMacroTable.INSTANCE)
- .addValue(ObjectMapper.class, objectMapper)
- .addValue(MetadataSupervisorManager.class, null)
- .addValue(IndexerMetadataStorageCoordinator.class, null)
- .addValue(MaterializedViewTaskConfig.class, new MaterializedViewTaskConfig())
- .addValue(AuthorizerMapper.class, EasyMock.createMock(AuthorizerMapper.class))
- .addValue(ChatHandlerProvider.class, new NoopChatHandlerProvider())
- .addValue(SupervisorStateManagerConfig.class, new SupervisorStateManagerConfig())
- );
- }
-
- @Test
- public void testSupervisorSerialization() throws IOException
- {
- String supervisorStr = "{\n" +
- " \"type\" : \"derivativeDataSource\",\n" +
- " \"baseDataSource\": \"wikiticker\",\n" +
- " \"dimensionsSpec\":{\n" +
- " \"dimensions\" : [\n" +
- " \"isUnpatrolled\",\n" +
- " \"metroCode\",\n" +
- " \"namespace\",\n" +
- " \"page\",\n" +
- " \"regionIsoCode\",\n" +
- " \"regionName\",\n" +
- " \"user\"\n" +
- " ]\n" +
- " },\n" +
- " \"metricsSpec\" : [\n" +
- " {\n" +
- " \"name\" : \"count\",\n" +
- " \"type\" : \"count\"\n" +
- " },\n" +
- " {\n" +
- " \"name\" : \"added\",\n" +
- " \"type\" : \"longSum\",\n" +
- " \"fieldName\" : \"added\"\n" +
- " }\n" +
- " ],\n" +
- " \"tuningConfig\": {\n" +
- " \"type\" : \"hadoop\"\n" +
- " }\n" +
- "}";
- MaterializedViewSupervisorSpec expected = new MaterializedViewSupervisorSpec(
- "wikiticker",
- new DimensionsSpec(
- Lists.newArrayList(
- new StringDimensionSchema("isUnpatrolled"),
- new StringDimensionSchema("metroCode"),
- new StringDimensionSchema("namespace"),
- new StringDimensionSchema("page"),
- new StringDimensionSchema("regionIsoCode"),
- new StringDimensionSchema("regionName"),
- new StringDimensionSchema("user")
- )
- ),
- new AggregatorFactory[]{
- new CountAggregatorFactory("count"),
- new LongSumAggregatorFactory("added", "added")
- },
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- null,
- false,
- objectMapper,
- null,
- null,
- null,
- null,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- new NoopChatHandlerProvider(),
- new SupervisorStateManagerConfig()
- );
- MaterializedViewSupervisorSpec spec = objectMapper.readValue(supervisorStr, MaterializedViewSupervisorSpec.class);
- Assert.assertEquals(expected.getBaseDataSource(), spec.getBaseDataSource());
- Assert.assertEquals(expected.getId(), spec.getId());
- Assert.assertEquals(expected.getDataSourceName(), spec.getDataSourceName());
- Assert.assertEquals(expected.getDimensions(), spec.getDimensions());
- Assert.assertEquals(expected.getMetrics(), spec.getMetrics());
- }
-
- @Test
- public void testMaterializedViewSupervisorSpecCreated()
- {
- Exception ex = null;
-
- try {
- MaterializedViewSupervisorSpec spec = new MaterializedViewSupervisorSpec(
- "wikiticker",
- new DimensionsSpec(
- Lists.newArrayList(
- new StringDimensionSchema("isUnpatrolled"),
- new StringDimensionSchema("metroCode"),
- new StringDimensionSchema("namespace"),
- new StringDimensionSchema("page"),
- new StringDimensionSchema("regionIsoCode"),
- new StringDimensionSchema("regionName"),
- new StringDimensionSchema("user")
- )
- ),
- new AggregatorFactory[]{
- new CountAggregatorFactory("count"),
- new LongSumAggregatorFactory("added", "added")
- },
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- null,
- false,
- objectMapper,
- null,
- null,
- null,
- null,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- new NoopChatHandlerProvider(),
- new SupervisorStateManagerConfig()
- );
- Supervisor supervisor = spec.createSupervisor();
- Assert.assertTrue(supervisor instanceof MaterializedViewSupervisor);
-
- SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor);
- Assert.assertNull(autoscaler);
- }
- catch (Exception e) {
- ex = e;
- }
-
- Assert.assertNull(ex);
- }
-
- @Test
- public void testSuspendResuume() throws IOException
- {
- String supervisorStr = "{\n" +
- " \"type\" : \"derivativeDataSource\",\n" +
- " \"baseDataSource\": \"wikiticker\",\n" +
- " \"dimensionsSpec\":{\n" +
- " \"dimensions\" : [\n" +
- " \"isUnpatrolled\",\n" +
- " \"metroCode\",\n" +
- " \"namespace\",\n" +
- " \"page\",\n" +
- " \"regionIsoCode\",\n" +
- " \"regionName\",\n" +
- " \"user\"\n" +
- " ]\n" +
- " },\n" +
- " \"metricsSpec\" : [\n" +
- " {\n" +
- " \"name\" : \"count\",\n" +
- " \"type\" : \"count\"\n" +
- " },\n" +
- " {\n" +
- " \"name\" : \"added\",\n" +
- " \"type\" : \"longSum\",\n" +
- " \"fieldName\" : \"added\"\n" +
- " }\n" +
- " ],\n" +
- " \"tuningConfig\": {\n" +
- " \"type\" : \"hadoop\"\n" +
- " }\n" +
- "}";
-
- MaterializedViewSupervisorSpec spec = objectMapper.readValue(supervisorStr, MaterializedViewSupervisorSpec.class);
- Assert.assertFalse(spec.isSuspended());
-
- String suspendedSerialized = objectMapper.writeValueAsString(spec.createSuspendedSpec());
- MaterializedViewSupervisorSpec suspendedSpec = objectMapper.readValue(
- suspendedSerialized,
- MaterializedViewSupervisorSpec.class
- );
- Assert.assertTrue(suspendedSpec.isSuspended());
-
- String runningSerialized = objectMapper.writeValueAsString(spec.createRunningSpec());
- MaterializedViewSupervisorSpec runningSpec = objectMapper.readValue(
- runningSerialized,
- MaterializedViewSupervisorSpec.class
- );
- Assert.assertFalse(runningSpec.isSuspended());
- }
-
- @Test
- public void testEmptyBaseDataSource()
- {
- expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
- expectedException.expectMessage(
- "baseDataSource cannot be null or empty. Please provide a baseDataSource."
- );
- //noinspection ResultOfObjectAllocationIgnored (this method call will trigger the expected exception)
- new MaterializedViewSupervisorSpec(
- "",
- new DimensionsSpec(
- Lists.newArrayList(
- new StringDimensionSchema("isUnpatrolled"),
- new StringDimensionSchema("metroCode"),
- new StringDimensionSchema("namespace"),
- new StringDimensionSchema("page"),
- new StringDimensionSchema("regionIsoCode"),
- new StringDimensionSchema("regionName"),
- new StringDimensionSchema("user")
- )
- ),
- new AggregatorFactory[]{
- new CountAggregatorFactory("count"),
- new LongSumAggregatorFactory("added", "added")
- },
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- null,
- false,
- objectMapper,
- null,
- null,
- null,
- null,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- new NoopChatHandlerProvider(),
- new SupervisorStateManagerConfig()
- );
- }
-
- @Test
- public void testNullBaseDataSource()
- {
- expectedException.expect(CoreMatchers.instanceOf(IllegalArgumentException.class));
- expectedException.expectMessage(
- "baseDataSource cannot be null or empty. Please provide a baseDataSource."
- );
- //noinspection ResultOfObjectAllocationIgnored (this method call will trigger the expected exception)
- new MaterializedViewSupervisorSpec(
- null,
- new DimensionsSpec(
- Lists.newArrayList(
- new StringDimensionSchema("isUnpatrolled"),
- new StringDimensionSchema("metroCode"),
- new StringDimensionSchema("namespace"),
- new StringDimensionSchema("page"),
- new StringDimensionSchema("regionIsoCode"),
- new StringDimensionSchema("regionName"),
- new StringDimensionSchema("user")
- )
- ),
- new AggregatorFactory[]{
- new CountAggregatorFactory("count"),
- new LongSumAggregatorFactory("added", "added")
- },
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- null,
- false,
- objectMapper,
- null,
- null,
- null,
- null,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- new NoopChatHandlerProvider(),
- new SupervisorStateManagerConfig()
- );
- }
-}
diff --git a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java b/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java
deleted file mode 100644
index 212a1e8529d..00000000000
--- a/extensions-contrib/materialized-view-maintenance/src/test/java/org/apache/druid/indexing/materializedview/MaterializedViewSupervisorTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.indexing.materializedview;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
-import junit.framework.AssertionFailedError;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.StringDimensionSchema;
-import org.apache.druid.error.EntryAlreadyExists;
-import org.apache.druid.indexer.HadoopIOConfig;
-import org.apache.druid.indexer.HadoopIndexTask;
-import org.apache.druid.indexer.HadoopIngestionSpec;
-import org.apache.druid.indexer.HadoopTaskConfig;
-import org.apache.druid.indexer.HadoopTuningConfig;
-import org.apache.druid.indexer.TaskStatus;
-import org.apache.druid.indexing.overlord.IndexerMetadataStorageCoordinator;
-import org.apache.druid.indexing.overlord.TaskMaster;
-import org.apache.druid.indexing.overlord.TaskQueue;
-import org.apache.druid.indexing.overlord.TaskStorage;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorStateManagerConfig;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
-import org.apache.druid.metadata.MetadataSupervisorManager;
-import org.apache.druid.metadata.TestDerbyConnector;
-import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
-import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.segment.TestHelper;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
-import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage;
-import org.apache.druid.segment.metadata.SegmentSchemaManager;
-import org.apache.druid.segment.realtime.ChatHandlerProvider;
-import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector;
-import org.apache.druid.server.metrics.NoopServiceEmitter;
-import org.apache.druid.server.security.AuthorizerMapper;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.HashBasedNumberedShardSpec;
-import org.easymock.EasyMock;
-import org.joda.time.Interval;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedMap;
-
-public class MaterializedViewSupervisorTest
-{
- @Rule
- public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule
- = new TestDerbyConnector.DerbyConnectorRule();
-
- private TaskStorage taskStorage;
- private TaskMaster taskMaster;
- private IndexerMetadataStorageCoordinator indexerMetadataStorageCoordinator;
- private MetadataSupervisorManager metadataSupervisorManager;
- private TaskQueue taskQueue;
- private MaterializedViewSupervisor supervisor;
- private String derivativeDatasourceName;
- private MaterializedViewSupervisorSpec spec;
- private final ObjectMapper objectMapper = TestHelper.makeJsonMapper();
- private SegmentSchemaManager segmentSchemaManager;
-
- @Before
- public void setUp()
- {
- TestDerbyConnector derbyConnector = derbyConnectorRule.getConnector();
- derbyConnector.createDataSourceTable();
- derbyConnector.createSegmentSchemasTable();
- derbyConnector.createSegmentTable();
- taskStorage = EasyMock.createMock(TaskStorage.class);
- taskMaster = EasyMock.createMock(TaskMaster.class);
- segmentSchemaManager = new SegmentSchemaManager(
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- objectMapper,
- derbyConnector
- );
- indexerMetadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(
- new SqlSegmentMetadataTransactionFactory(
- objectMapper,
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- derbyConnector,
- new TestDruidLeaderSelector(),
- NoopSegmentMetadataCache.instance(),
- NoopServiceEmitter.instance()
- ),
- objectMapper,
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- derbyConnector,
- segmentSchemaManager,
- CentralizedDatasourceSchemaConfig.create(),
- new HeapMemoryIndexingStateStorage()
- );
- metadataSupervisorManager = EasyMock.createMock(MetadataSupervisorManager.class);
- taskQueue = EasyMock.createMock(TaskQueue.class);
-
- objectMapper.registerSubtypes(new NamedType(HashBasedNumberedShardSpec.class, "hashed"));
- spec = new MaterializedViewSupervisorSpec(
- "base",
- new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))),
- new AggregatorFactory[]{new LongSumAggregatorFactory("m1", "m1")},
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- Collections.singletonMap("maxTaskCount", 2),
- false,
- objectMapper,
- taskMaster,
- taskStorage,
- metadataSupervisorManager,
- indexerMetadataStorageCoordinator,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- EasyMock.createMock(ChatHandlerProvider.class),
- new SupervisorStateManagerConfig()
- );
- derivativeDatasourceName = spec.getDataSourceName();
- supervisor = (MaterializedViewSupervisor) spec.createSupervisor();
- }
-
- @Test
- public void testCheckSegments()
- {
- List<DataSegment> baseSegments = createBaseSegments();
- Set<DataSegment> derivativeSegments = Sets.newHashSet(createDerivativeSegments());
-
- final Interval day1 = baseSegments.get(0).getInterval();
- final Interval day2 = new Interval(day1.getStart().plusDays(1), day1.getEnd().plusDays(1));
-
- indexerMetadataStorageCoordinator.commitSegments(new HashSet<>(baseSegments), null);
- indexerMetadataStorageCoordinator.commitSegments(derivativeSegments, null);
- EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
- EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
- EasyMock.expect(taskStorage.getActiveTasks()).andReturn(ImmutableList.of()).anyTimes();
-
- Pair<SortedMap<Interval, String>, Map<Interval, List<DataSegment>>> toBuildInterval
- = supervisor.checkSegments();
-
- Map<Interval, List<DataSegment>> expectedSegments = ImmutableMap.of(
- day1, Collections.singletonList(baseSegments.get(0)),
- day2, Collections.singletonList(baseSegments.get(1))
- );
- Assert.assertEquals(Collections.singleton(day1), toBuildInterval.lhs.keySet());
- Assert.assertEquals(expectedSegments, toBuildInterval.rhs);
- }
-
- @Test
- public void testSubmitTasksDoesNotFailIfTaskAlreadyExists()
- {
- Set<DataSegment> baseSegments = Sets.newHashSet(createBaseSegments());
- Set<DataSegment> derivativeSegments = Sets.newHashSet(createDerivativeSegments());
-
- indexerMetadataStorageCoordinator.commitSegments(baseSegments, null);
- indexerMetadataStorageCoordinator.commitSegments(derivativeSegments, null);
- EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
- EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
- EasyMock.expect(taskStorage.getActiveTasks()).andReturn(ImmutableList.of()).anyTimes();
-
- EasyMock.expect(taskQueue.add(EasyMock.anyObject()))
- .andThrow(EntryAlreadyExists.exception("Task ID already exists"));
-
- EasyMock.replay(taskMaster, taskStorage, taskQueue);
-
- supervisor.checkSegmentsAndSubmitTasks();
-
- EasyMock.verify(taskMaster, taskStorage, taskQueue);
- }
-
- @Test
- public void testSubmitTasksFailsIfTaskCannotBeAdded()
- {
- Set<DataSegment> baseSegments = Sets.newHashSet(createBaseSegments());
- Set<DataSegment> derivativeSegments = Sets.newHashSet(createDerivativeSegments());
-
- indexerMetadataStorageCoordinator.commitSegments(baseSegments, null);
- indexerMetadataStorageCoordinator.commitSegments(derivativeSegments, null);
- EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
- EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
- EasyMock.expect(taskStorage.getActiveTasks()).andReturn(ImmutableList.of()).anyTimes();
-
- EasyMock.expect(taskQueue.add(EasyMock.anyObject()))
- .andThrow(new ISE("Could not add task"));
-
- EasyMock.replay(taskMaster, taskStorage, taskQueue);
-
- ISE exception = Assert.assertThrows(
- ISE.class,
- () -> supervisor.checkSegmentsAndSubmitTasks()
- );
- Assert.assertEquals("Could not add task", exception.getMessage());
-
- EasyMock.verify(taskMaster, taskStorage, taskQueue);
- }
-
- @Test
- public void testCheckSegmentsAndSubmitTasks()
- {
- Set<DataSegment> baseSegments = Collections.singleton(createBaseSegments().get(0));
- indexerMetadataStorageCoordinator.commitSegments(baseSegments, null);
- EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
- EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
- EasyMock.expect(taskStorage.getActiveTasks()).andReturn(ImmutableList.of()).anyTimes();
- EasyMock.expect(taskStorage.getStatus("test_task1"))
- .andReturn(Optional.of(TaskStatus.failure("test_task1", "Dummy task status failure err message")))
- .anyTimes();
- EasyMock.expect(taskStorage.getStatus("test_task2"))
- .andReturn(Optional.of(TaskStatus.running("test_task2")))
- .anyTimes();
- EasyMock.replay(taskStorage);
-
- Pair<Map<Interval, HadoopIndexTask>, Map<Interval, String>> runningTasksPair = supervisor.getRunningTasks();
- Map<Interval, HadoopIndexTask> runningTasks = runningTasksPair.lhs;
- Map<Interval, String> runningVersion = runningTasksPair.rhs;
-
- DataSchema dataSchema = DataSchema.builder()
- .withDataSource("test_datasource")
- .withObjectMapper(objectMapper)
- .build();
- HadoopIOConfig hadoopIOConfig = new HadoopIOConfig(new HashMap<>(), null, null);
- HadoopIngestionSpec spec = new HadoopIngestionSpec(dataSchema, hadoopIOConfig, null);
- HadoopIndexTask task1 = new HadoopIndexTask(
- "test_task1",
- spec,
- null,
- null,
- null,
- objectMapper,
- null,
- null,
- null,
- new HadoopTaskConfig(null, null)
- );
- runningTasks.put(Intervals.of("2015-01-01T00Z/2015-01-02T00Z"), task1);
- runningVersion.put(Intervals.of("2015-01-01T00Z/2015-01-02T00Z"), "test_version1");
-
- HadoopIndexTask task2 = new HadoopIndexTask(
- "test_task2",
- spec,
- null,
- null,
- null,
- objectMapper,
- null,
- null,
- null,
- new HadoopTaskConfig(null, null)
- );
- runningTasks.put(Intervals.of("2015-01-02T00Z/2015-01-03T00Z"), task2);
- runningVersion.put(Intervals.of("2015-01-02T00Z/2015-01-03T00Z"), "test_version2");
-
- supervisor.checkSegmentsAndSubmitTasks();
-
- Map<Interval, HadoopIndexTask> expectedRunningTasks = new HashMap<>();
- Map<Interval, String> expectedRunningVersion = new HashMap<>();
- expectedRunningTasks.put(Intervals.of("2015-01-02T00Z/2015-01-03T00Z"), task2);
- expectedRunningVersion.put(Intervals.of("2015-01-02T00Z/2015-01-03T00Z"), "test_version2");
-
- Assert.assertEquals(expectedRunningTasks, runningTasks);
- Assert.assertEquals(expectedRunningVersion, runningVersion);
- }
-
- @Test
- public void testCreateTaskSucceeds()
- {
- List<DataSegment> baseSegments = createBaseSegments().subList(0, 1);
-
- HadoopIndexTask task = spec.createTask(
- Intervals.of("2015-01-02T00Z/2015-01-03T00Z"),
- "2015-01-03",
- baseSegments
- );
-
- Assert.assertNotNull(task);
- }
-
- @Test
- public void testSuspendedDoesNotRun()
- {
- MaterializedViewSupervisorSpec suspended = new MaterializedViewSupervisorSpec(
- "base",
- new DimensionsSpec(Collections.singletonList(new StringDimensionSchema("dim"))),
- new AggregatorFactory[]{new LongSumAggregatorFactory("m1", "m1")},
- HadoopTuningConfig.makeDefaultTuningConfig(),
- null,
- null,
- null,
- null,
- null,
- true,
- objectMapper,
- taskMaster,
- taskStorage,
- metadataSupervisorManager,
- indexerMetadataStorageCoordinator,
- new MaterializedViewTaskConfig(),
- EasyMock.createMock(AuthorizerMapper.class),
- EasyMock.createMock(ChatHandlerProvider.class),
- new SupervisorStateManagerConfig()
- );
- MaterializedViewSupervisor supervisor = (MaterializedViewSupervisor) suspended.createSupervisor();
-
- // mock IndexerSQLMetadataStorageCoordinator to ensure that retrieveDataSourceMetadata is not called
- // which will be true if truly suspended, since this is the first operation of the 'run' method otherwise
- IndexerSQLMetadataStorageCoordinator mock = EasyMock.createMock(IndexerSQLMetadataStorageCoordinator.class);
- EasyMock.expect(mock.retrieveDataSourceMetadata(suspended.getDataSourceName()))
- .andThrow(new AssertionFailedError())
- .anyTimes();
-
- EasyMock.replay(mock);
- supervisor.run();
- }
-
- private List<DataSegment> createBaseSegments()
- {
- return Arrays.asList(
- createSegment("base", "2015-01-01T00Z/2015-01-02T00Z", "2015-01-02"),
- createSegment("base", "2015-01-02T00Z/2015-01-03T00Z", "2015-01-03"),
- createSegment("base", "2015-01-03T00Z/2015-01-04T00Z", "2015-01-04")
- );
- }
-
- private List<DataSegment> createDerivativeSegments()
- {
- return Arrays.asList(
- createSegment(derivativeDatasourceName, "2015-01-01T00Z/2015-01-02T00Z", "2015-01-02"),
- createSegment(derivativeDatasourceName, "2015-01-02T00Z/2015-01-03T00Z", "3015-01-01")
- );
- }
-
- private DataSegment createSegment(String datasource, String interval, String version)
- {
- return new DataSegment(
- datasource,
- Intervals.of(interval),
- version,
- Collections.emptyMap(),
- Arrays.asList("dim1", "dim2"),
- Collections.singletonList("m2"),
- new HashBasedNumberedShardSpec(0, 1, 0, 1, null, null, null),
- 9,
- 1024
- );
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/pom.xml b/extensions-contrib/materialized-view-selection/pom.xml
deleted file mode 100644
index 003d5523212..00000000000
--- a/extensions-contrib/materialized-view-selection/pom.xml
+++ /dev/null
@@ -1,149 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>druid</artifactId>
- <groupId>org.apache.druid</groupId>
- <version>37.0.0-SNAPSHOT</version>
- <relativePath>../../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <groupId>org.apache.druid.extensions.contrib</groupId>
- <artifactId>materialized-view-selection</artifactId>
- <name>materialized-view-selection</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions.contrib</groupId>
- <artifactId>materialized-view-maintenance</artifactId>
- <version>${project.parent.version}</version>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-smile</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.jdbi</groupId>
- <artifactId>jdbi</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>jsr305</artifactId>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${project.parent.version}</version>
- <scope>test</scope>
- <type>test-jar</type>
- </dependency>
- <dependency>
- <groupId>org.easymock</groupId>
- <artifactId>easymock</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-test</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.hamcrest</groupId>
- <artifactId>hamcrest</artifactId>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.owasp</groupId>
- <artifactId>dependency-check-maven</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizer.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizer.java
deleted file mode 100644
index 823aecf6360..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSortedSet;
-import com.google.inject.Inject;
-import org.apache.druid.client.TimelineServerView;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.TableDataSource;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.topn.TopNQuery;
-import org.apache.druid.timeline.TimelineObjectHolder;
-import org.joda.time.Interval;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-public class DataSourceOptimizer
-{
- private final ReadWriteLock lock = new ReentrantReadWriteLock();
- private final TimelineServerView serverView;
- private ConcurrentHashMap<String, AtomicLong> derivativesHitCount = new ConcurrentHashMap<>();
- private ConcurrentHashMap<String, AtomicLong> totalCount = new ConcurrentHashMap<>();
- private ConcurrentHashMap<String, AtomicLong> hitCount = new ConcurrentHashMap<>();
- private ConcurrentHashMap<String, AtomicLong> costTime = new ConcurrentHashMap<>();
- private ConcurrentHashMap<String, ConcurrentHashMap<Set<String>, AtomicLong>> missFields = new ConcurrentHashMap<>();
-
- @Inject
- public DataSourceOptimizer(TimelineServerView serverView)
- {
- this.serverView = serverView;
- }
-
- /**
- * Do main work about materialized view selection: transform user query to one or more sub-queries.
- *
- * In the sub-query, the dataSource is the derivative of dataSource in user query, and sum of all sub-queries'
- * intervals equals the interval in user query
- *
- * Derived dataSource with smallest average data size per segment granularity have highest priority to replace the
- * datasource in user query
- *
- * @param query only TopNQuery/TimeseriesQuery/GroupByQuery can be optimized
- * @return a list of queries with specified derived dataSources and intervals
- */
- public List<Query> optimize(Query query)
- {
- long start = System.currentTimeMillis();
- // only topN/timeseries/groupby query can be optimized
- // only TableDataSource can be optimiezed
- if (!(query instanceof TopNQuery || query instanceof TimeseriesQuery || query instanceof GroupByQuery)
- || !(query.getDataSource() instanceof TableDataSource)) {
- return Collections.singletonList(query);
- }
- String datasourceName = ((TableDataSource) query.getDataSource()).getName();
- // get all derivatives for datasource in query. The derivatives set is sorted by average size of
- // per segment granularity.
- Set<DerivativeDataSource> derivatives = DerivativeDataSourceManager.getDerivatives(datasourceName);
-
- if (derivatives.isEmpty()) {
- return Collections.singletonList(query);
- }
- lock.readLock().lock();
- try {
- totalCount.computeIfAbsent(datasourceName, dsName -> new AtomicLong(0)).incrementAndGet();
- hitCount.putIfAbsent(datasourceName, new AtomicLong(0));
- AtomicLong costTimeOfDataSource = costTime.computeIfAbsent(datasourceName, dsName -> new AtomicLong(0));
-
- // get all fields which the query required
- Set<String> requiredFields = MaterializedViewUtils.getRequiredFields(query);
-
- Set<DerivativeDataSource> derivativesWithRequiredFields = new HashSet<>();
- for (DerivativeDataSource derivativeDataSource : derivatives) {
- derivativesHitCount.putIfAbsent(derivativeDataSource.getName(), new AtomicLong(0));
- if (derivativeDataSource.getColumns().containsAll(requiredFields)) {
- derivativesWithRequiredFields.add(derivativeDataSource);
- }
- }
- // if no derivatives contains all required dimensions, this materialized view selection failed.
- if (derivativesWithRequiredFields.isEmpty()) {
- missFields
- .computeIfAbsent(datasourceName, dsName -> new ConcurrentHashMap<>())
- .computeIfAbsent(requiredFields, rf -> new AtomicLong(0))
- .incrementAndGet();
- costTimeOfDataSource.addAndGet(System.currentTimeMillis() - start);
- return Collections.singletonList(query);
- }
-
- List<Query> queries = new ArrayList<>();
- List<Interval> remainingQueryIntervals = query.getIntervals();
-
- for (DerivativeDataSource derivativeDataSource : ImmutableSortedSet.copyOf(derivativesWithRequiredFields)) {
- TableDataSource tableDataSource = new TableDataSource(derivativeDataSource.getName());
- final List<Interval> derivativeIntervals = remainingQueryIntervals.stream()
- .flatMap(interval -> serverView
- .getTimeline(tableDataSource)
- .orElseThrow(() -> new ISE("No timeline for dataSource: %s", derivativeDataSource.getName()))
- .lookup(interval)
- .stream()
- .map(TimelineObjectHolder::getInterval)
- )
- .collect(Collectors.toList());
- // if the derivative does not contain any parts of intervals in the query, the derivative will
- // not be selected.
- if (derivativeIntervals.isEmpty()) {
- continue;
- }
-
- remainingQueryIntervals = MaterializedViewUtils.minus(remainingQueryIntervals, derivativeIntervals);
- queries.add(
- query.withDataSource(new TableDataSource(derivativeDataSource.getName()))
- .withQuerySegmentSpec(new MultipleIntervalSegmentSpec(derivativeIntervals))
- );
- derivativesHitCount.get(derivativeDataSource.getName()).incrementAndGet();
- if (remainingQueryIntervals.isEmpty()) {
- break;
- }
- }
-
- if (queries.isEmpty()) {
- costTime.get(datasourceName).addAndGet(System.currentTimeMillis() - start);
- return Collections.singletonList(query);
- }
-
- //after materialized view selection, the result of the remaining query interval will be computed based on
- // the original datasource.
- if (!remainingQueryIntervals.isEmpty()) {
- queries.add(query.withQuerySegmentSpec(new MultipleIntervalSegmentSpec(remainingQueryIntervals)));
- }
- hitCount.get(datasourceName).incrementAndGet();
- costTime.get(datasourceName).addAndGet(System.currentTimeMillis() - start);
- return queries;
- }
- finally {
- lock.readLock().unlock();
- }
- }
-
- public List<DataSourceOptimizerStats> getAndResetStats()
- {
- ImmutableMap<String, AtomicLong> derivativesHitCountSnapshot;
- ImmutableMap<String, AtomicLong> totalCountSnapshot;
- ImmutableMap<String, AtomicLong> hitCountSnapshot;
- ImmutableMap<String, AtomicLong> costTimeSnapshot;
- ImmutableMap<String, ConcurrentHashMap<Set<String>, AtomicLong>> missFieldsSnapshot;
- lock.writeLock().lock();
- try {
- derivativesHitCountSnapshot = ImmutableMap.copyOf(derivativesHitCount);
- totalCountSnapshot = ImmutableMap.copyOf(totalCount);
- hitCountSnapshot = ImmutableMap.copyOf(hitCount);
- costTimeSnapshot = ImmutableMap.copyOf(costTime);
- missFieldsSnapshot = ImmutableMap.copyOf(missFields);
- derivativesHitCount.clear();
- totalCount.clear();
- hitCount.clear();
- costTime.clear();
- missFields.clear();
- }
- finally {
- lock.writeLock().unlock();
- }
- List<DataSourceOptimizerStats> stats = new ArrayList<>();
- Map<String, Set<DerivativeDataSource>> baseToDerivatives = DerivativeDataSourceManager.getAllDerivatives();
- for (Map.Entry<String, Set<DerivativeDataSource>> entry : baseToDerivatives.entrySet()) {
- Map<String, Long> derivativesStat = new HashMap<>();
- for (DerivativeDataSource derivative : entry.getValue()) {
- derivativesStat.put(
- derivative.getName(),
- derivativesHitCountSnapshot.getOrDefault(derivative.getName(), new AtomicLong(0)).get()
- );
- }
- stats.add(
- new DataSourceOptimizerStats(
- entry.getKey(),
- hitCountSnapshot.getOrDefault(entry.getKey(), new AtomicLong(0)).get(),
- totalCountSnapshot.getOrDefault(entry.getKey(), new AtomicLong(0)).get(),
- costTimeSnapshot.getOrDefault(entry.getKey(), new AtomicLong(0)).get(),
- missFieldsSnapshot.getOrDefault(entry.getKey(), new ConcurrentHashMap<>()),
- derivativesStat
- )
- );
- }
- return stats;
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
deleted file mode 100644
index 03abc99a165..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitor.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.google.inject.Inject;
-import org.apache.druid.java.util.emitter.service.ServiceEmitter;
-import org.apache.druid.java.util.emitter.service.ServiceMetricEvent;
-import org.apache.druid.java.util.metrics.AbstractMonitor;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class DataSourceOptimizerMonitor extends AbstractMonitor
-{
- private final DataSourceOptimizer optimizer;
-
- @Inject
- public DataSourceOptimizerMonitor(DataSourceOptimizer optimizer)
- {
- this.optimizer = optimizer;
- }
-
- @Override
- public boolean doMonitor(ServiceEmitter emitter)
- {
- final List<DataSourceOptimizerStats> stats = optimizer.getAndResetStats();
- for (DataSourceOptimizerStats stat : stats) {
- final ServiceMetricEvent.Builder builder = new ServiceMetricEvent.Builder();
- builder.setDimension("dataSource", stat.getBase());
- emitter.emit(builder.setMetric("/materialized/view/query/totalNum", stat.getTotalcount()));
- emitter.emit(builder.setMetric("/materialized/view/query/hits", stat.getHitcount()));
- emitter.emit(builder.setMetric("/materialized/view/query/hitRate", stat.getHitRate()));
- emitter.emit(builder.setMetric("/materialized/view/select/avgCostMS", stat.getOptimizerCost()));
- Map<String, Long> derivativesStats = stat.getDerivativesHitCount();
- for (Map.Entry<String, Long> derivative : derivativesStats.entrySet()) {
- builder.setDimension("derivative", derivative.getKey());
- emitter.emit(builder.setMetric("/materialized/view/derivative/numSelected", derivative.getValue()));
- }
- final ServiceMetricEvent.Builder builder2 = new ServiceMetricEvent.Builder();
- builder2.setDimension("dataSource", stat.getBase());
- for (Set<String> fields : stat.getMissFields().keySet()) {
- builder2.setDimension("fields", fields.toString());
- emitter.emit(builder2.setMetric("/materialized/view/missNum", stat.getMissFields().get(fields).get()));
- }
- }
- return true;
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerStats.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerStats.java
deleted file mode 100644
index 2c86df8fad7..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DataSourceOptimizerStats.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
-
-public class DataSourceOptimizerStats
-{
- private final String base;
- private final long hitcount;
- private final long totalcount;
- private final long optimizerCost;
- private final Map<Set<String>, AtomicLong> missFields;
- private final Map<String, Long> derivativesHitCount;
-
- public DataSourceOptimizerStats(
- String base,
- long hitcount,
- long totalcount,
- long optimizerCost,
- Map<Set<String>, AtomicLong> missFields,
- Map<String, Long> derivativesHitCount
- )
- {
- this.base = base;
- this.hitcount = hitcount;
- this.totalcount = totalcount;
- this.optimizerCost = optimizerCost;
- this.missFields = missFields;
- this.derivativesHitCount = derivativesHitCount;
- }
-
- public Map<Set<String>, AtomicLong> getMissFields()
- {
- return missFields;
- }
-
- public String getBase()
- {
- return base;
- }
-
- public long getHitcount()
- {
- return hitcount;
- }
-
- public long getTotalcount()
- {
- return totalcount;
- }
-
- public double getOptimizerCost()
- {
- if (totalcount == 0L) {
- return 0;
- }
- return ((double) optimizerCost) / totalcount;
- }
-
- public double getHitRate()
- {
- if (totalcount == 0L) {
- return 0;
- }
- return ((double) hitcount) / totalcount;
- }
-
- public Map<String, Long> getDerivativesHitCount()
- {
- return derivativesHitCount;
- }
-
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSource.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSource.java
deleted file mode 100644
index be9c5f833cb..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSource.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.google.common.base.Preconditions;
-
-import java.util.Objects;
-import java.util.Set;
-
-public class DerivativeDataSource implements Comparable<DerivativeDataSource>
-{
- private final String name;
- private final String baseDataSource;
- private final Set<String> columns;
- private final long avgSizeBasedGranularity;
-
- public DerivativeDataSource(String name, String baseDataSource, Set<String> columns, long size)
- {
- this.name = Preconditions.checkNotNull(name, "name");
- this.baseDataSource = Preconditions.checkNotNull(baseDataSource, "baseDataSource");
- this.columns = Preconditions.checkNotNull(columns, "columns");
- this.avgSizeBasedGranularity = size;
- }
-
- public String getName()
- {
- return name;
- }
-
- public String getBaseDataSource()
- {
- return baseDataSource;
- }
-
- public Set<String> getColumns()
- {
- return columns;
- }
-
- public long getAvgSizeBasedGranularity()
- {
- return avgSizeBasedGranularity;
- }
-
- @Override
- public int compareTo(DerivativeDataSource o)
- {
- if (this.avgSizeBasedGranularity > o.getAvgSizeBasedGranularity()) {
- return 1;
- } else if (this.avgSizeBasedGranularity == o.getAvgSizeBasedGranularity()) {
- return 0;
- } else {
- return -1;
- }
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (o == null) {
- return false;
- }
- if (!(o instanceof DerivativeDataSource)) {
- return false;
- }
- DerivativeDataSource that = (DerivativeDataSource) o;
- return name.equals(that.getName())
- && baseDataSource.equals(that.getBaseDataSource())
- && columns.equals(that.getColumns());
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(name, baseDataSource, columns);
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSourceManager.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSourceManager.java
deleted file mode 100644
index 82ad698d8bd..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/DerivativeDataSourceManager.java
+++ /dev/null
@@ -1,254 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Supplier;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.ListeningScheduledExecutorService;
-import com.google.common.util.concurrent.MoreExecutors;
-import com.google.inject.Inject;
-import org.apache.druid.guice.ManageLifecycle;
-import org.apache.druid.indexing.materializedview.DerivativeDataSourceMetadata;
-import org.apache.druid.indexing.overlord.DataSourceMetadata;
-import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.StringUtils;
-import org.apache.druid.java.util.common.concurrent.Execs;
-import org.apache.druid.java.util.common.jackson.JacksonUtils;
-import org.apache.druid.java.util.common.lifecycle.LifecycleStart;
-import org.apache.druid.java.util.common.lifecycle.LifecycleStop;
-import org.apache.druid.java.util.emitter.EmittingLogger;
-import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.SQLMetadataConnector;
-import org.apache.druid.timeline.DataSegment;
-import org.joda.time.Duration;
-import org.joda.time.Interval;
-import org.skife.jdbi.v2.Handle;
-import org.skife.jdbi.v2.StatementContext;
-import org.skife.jdbi.v2.tweak.HandleCallback;
-
-import java.sql.ResultSet;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-
-/**
- * Read and store derivatives information from dataSource table frequently.
- * When optimize query, DerivativesManager offers the information about derivatives.
- */
-@ManageLifecycle
-public class DerivativeDataSourceManager
-{
- private static final EmittingLogger log = new EmittingLogger(DerivativeDataSourceManager.class);
- private static final AtomicReference<ConcurrentHashMap<String, SortedSet<DerivativeDataSource>>> DERIVATIVES_REF =
- new AtomicReference<>(new ConcurrentHashMap<>());
- private final MaterializedViewConfig config;
- private final Supplier<MetadataStorageTablesConfig> dbTables;
- private final SQLMetadataConnector connector;
- private final ObjectMapper objectMapper;
- private final Object lock = new Object();
-
- private boolean started = false;
- private ListeningScheduledExecutorService exec = null;
- private ListenableFuture<?> future = null;
-
- @Inject
- public DerivativeDataSourceManager(
- MaterializedViewConfig config,
- Supplier<MetadataStorageTablesConfig> dbTables,
- ObjectMapper objectMapper,
- SQLMetadataConnector connector
- )
- {
- this.config = config;
- this.dbTables = dbTables;
- this.objectMapper = objectMapper;
- this.connector = connector;
- }
-
- @LifecycleStart
- public void start()
- {
- log.info("starting derivatives manager.");
- synchronized (lock) {
- if (started) {
- return;
- }
- exec = MoreExecutors.listeningDecorator(Execs.scheduledSingleThreaded("DerivativeDataSourceManager-Exec-%d"));
- final Duration delay = config.getPollDuration().toStandardDuration();
- future = exec.scheduleWithFixedDelay(
- new Runnable() {
- @Override
- public void run()
- {
- try {
- updateDerivatives();
- }
- catch (Exception e) {
- log.makeAlert(e, "uncaught exception in derivatives manager updating thread").emit();
- }
- }
- },
- 0,
- delay.getMillis(),
- TimeUnit.MILLISECONDS
- );
- started = true;
- }
- log.info("Derivatives manager started.");
- }
-
- @LifecycleStop
- public void stop()
- {
- synchronized (lock) {
- if (!started) {
- return;
- }
- started = false;
- future.cancel(true);
- future = null;
- DERIVATIVES_REF.set(new ConcurrentHashMap<>());
- exec.shutdownNow();
- exec = null;
- }
- }
-
- public static ImmutableSet<DerivativeDataSource> getDerivatives(String datasource)
- {
- return ImmutableSet.copyOf(DERIVATIVES_REF.get().getOrDefault(datasource, new TreeSet<>()));
- }
-
- public static ImmutableMap<String, Set<DerivativeDataSource>> getAllDerivatives()
- {
- return ImmutableMap.copyOf(DERIVATIVES_REF.get());
- }
-
- private void updateDerivatives()
- {
- List<Pair<String, DerivativeDataSourceMetadata>> derivativesInDatabase = connector.retryWithHandle(
- handle ->
- handle
- .createQuery(
- StringUtils.format(
- "SELECT DISTINCT dataSource,commit_metadata_payload FROM %1$s",
- dbTables.get().getDataSourceTable()
- )
- )
- .map((int index, ResultSet r, StatementContext ctx) -> {
- String datasourceName = r.getString("dataSource");
- DataSourceMetadata payload = JacksonUtils.readValue(
- objectMapper,
- r.getBytes("commit_metadata_payload"),
- DataSourceMetadata.class);
- if (!(payload instanceof DerivativeDataSourceMetadata)) {
- return null;
- }
- DerivativeDataSourceMetadata metadata = (DerivativeDataSourceMetadata) payload;
- return new Pair<>(datasourceName, metadata);
- })
- .list()
- );
-
- List<DerivativeDataSource> derivativeDataSources = derivativesInDatabase.parallelStream()
- .filter(data -> data != null)
- .map(derivatives -> {
- String name = derivatives.lhs;
- DerivativeDataSourceMetadata metadata = derivatives.rhs;
- String baseDataSource = metadata.getBaseDataSource();
- long avgSizePerGranularity = getAvgSizePerGranularity(name);
- log.info("find derivatives: {bases=%s, derivative=%s, dimensions=%s, metrics=%s, avgSize=%s}",
- baseDataSource, name, metadata.getDimensions(), metadata.getMetrics(), avgSizePerGranularity);
- return new DerivativeDataSource(name, baseDataSource, metadata.getColumns(), avgSizePerGranularity);
- })
- .filter(derivatives -> derivatives.getAvgSizeBasedGranularity() > 0)
- .collect(Collectors.toList());
-
- ConcurrentHashMap<String, SortedSet<DerivativeDataSource>> newDerivatives = new ConcurrentHashMap<>();
- for (DerivativeDataSource derivative : derivativeDataSources) {
- newDerivatives.computeIfAbsent(derivative.getBaseDataSource(), ds -> new TreeSet<>()).add(derivative);
- }
- ConcurrentHashMap<String, SortedSet<DerivativeDataSource>> current;
- do {
- current = DERIVATIVES_REF.get();
- } while (!DERIVATIVES_REF.compareAndSet(current, newDerivatives));
- }
-
- /**
- * calculate the average data size per segment granularity for a given datasource.
- *
- * e.g. for a datasource, there're 5 segments as follows,
- * interval = "2018-04-01/2017-04-02", segment size = 1024 * 1024 * 2
- * interval = "2018-04-01/2017-04-02", segment size = 1024 * 1024 * 2
- * interval = "2018-04-02/2017-04-03", segment size = 1024 * 1024 * 1
- * interval = "2018-04-02/2017-04-03", segment size = 1024 * 1024 * 1
- * interval = "2018-04-02/2017-04-03", segment size = 1024 * 1024 * 1
- * Then, we get interval number = 2, total segment size = 1024 * 1024 * 7
- * At last, return the result 1024 * 1024 * 7 / 2 = 1024 * 1024 * 3.5
- *
- * @param datasource
- * @return average data size per segment granularity
- */
- private long getAvgSizePerGranularity(String datasource)
- {
- return connector.retryWithHandle(
- new HandleCallback<>() {
- Set<Interval> intervals = new HashSet<>();
- long totalSize = 0;
- @Override
- public Long withHandle(Handle handle)
- {
- handle.createQuery(
- StringUtils.format("SELECT start,%1$send%1$s,payload FROM %2$s WHERE used = true AND dataSource = :dataSource",
- connector.getQuoteString(), dbTables.get().getSegmentsTable()
- )
- )
- .bind("dataSource", datasource)
- .map(
- (int index, ResultSet r, StatementContext ctx) -> {
- intervals.add(
- Intervals.utc(
- DateTimes.of(r.getString("start")).getMillis(),
- DateTimes.of(r.getString("end")).getMillis()
- )
- );
- DataSegment segment =
- JacksonUtils.readValue(objectMapper, r.getBytes("payload"), DataSegment.class);
- totalSize += segment.getSize();
- return null;
- }
- )
- .list();
- return intervals.isEmpty() ? 0L : totalSize / intervals.size();
- }
- }
- );
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewConfig.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewConfig.java
deleted file mode 100644
index ca58daf3c32..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewConfig.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.joda.time.Period;
-
-public class MaterializedViewConfig
-{
- @JsonProperty
- private Period pollDuration = new Period("PT1M");
-
- public Period getPollDuration()
- {
- return pollDuration;
- }
-
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java
deleted file mode 100644
index ec061370a4d..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQuery.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.annotation.JacksonInject;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Ordering;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.query.BaseQuery;
-import org.apache.druid.query.DataSource;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryRunner;
-import org.apache.druid.query.QuerySegmentWalker;
-import org.apache.druid.query.filter.DimFilter;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.spec.QuerySegmentSpec;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.topn.TopNQuery;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Duration;
-import org.joda.time.Interval;
-
-import javax.annotation.Nullable;
-
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * MaterializedViewQuery helps to do materialized view selection automatically.
- *
- * Each MaterializedViewQuery contains a real query which type can be topn, timeseries or groupBy.
- * The real query will be optimized based on its dataSources and intervals. It will be converted into one or more
- * sub-queries, in which dataSources and intervals are replaced by derived dataSources and related sub-intervals.
- *
- * Derived dataSources always have less dimensions, but contains all dimensions which real query required.
- */
-public class MaterializedViewQuery<T> implements Query<T>
-{
- public static final String TYPE = "view";
- private final Query query;
- private final DataSourceOptimizer optimizer;
-
- @JsonCreator
- public MaterializedViewQuery(
- @JsonProperty("query") Query query,
- @JacksonInject DataSourceOptimizer optimizer
- )
- {
- Preconditions.checkArgument(
- query instanceof TopNQuery || query instanceof TimeseriesQuery || query instanceof GroupByQuery,
- "Only topN/timeseries/groupby query are supported"
- );
- this.query = query;
- this.optimizer = optimizer;
- }
-
- @JsonProperty("query")
- public Query getQuery()
- {
- return query;
- }
-
- public DataSourceOptimizer getOptimizer()
- {
- return optimizer;
- }
-
- @Override
- public DataSource getDataSource()
- {
- return query.getDataSource();
- }
-
- @Override
- public boolean hasFilters()
- {
- return query.hasFilters();
- }
-
- @Override
- public DimFilter getFilter()
- {
- return query.getFilter();
- }
-
- @Override
- public String getType()
- {
- return query.getType();
- }
-
- @Override
- public QueryRunner<T> getRunner(QuerySegmentWalker walker)
- {
- return ((BaseQuery) query).getQuerySegmentSpec().lookup(this, walker);
- }
-
- @Override
- public List<Interval> getIntervals()
-
- {
- return query.getIntervals();
- }
-
- @Override
- public Duration getDuration()
- {
- return query.getDuration();
- }
-
- @Override
- public Granularity getGranularity()
- {
- return query.getGranularity();
- }
-
- @Override
- public DateTimeZone getTimezone()
- {
- return query.getTimezone();
- }
-
- @Override
- public Map<String, Object> getContext()
- {
- return query.getContext();
- }
-
- @Override
- public Ordering<T> getResultOrdering()
- {
- return query.getResultOrdering();
- }
-
- @Override
- public MaterializedViewQuery withOverriddenContext(Map<String, Object> contextOverride)
- {
- return new MaterializedViewQuery(query.withOverriddenContext(contextOverride), optimizer);
- }
-
- @Override
- public MaterializedViewQuery withQuerySegmentSpec(QuerySegmentSpec spec)
- {
- return new MaterializedViewQuery(query.withQuerySegmentSpec(spec), optimizer);
- }
-
- @Override
- public MaterializedViewQuery withId(String id)
- {
- return new MaterializedViewQuery(query.withId(id), optimizer);
- }
-
- @Override
- public String getId()
- {
- return query.getId();
- }
-
- @Override
- public Query<T> withSubQueryId(String subQueryId)
- {
- return new MaterializedViewQuery<>(query.withSubQueryId(subQueryId), optimizer);
- }
-
- @Nullable
- @Override
- public String getSubQueryId()
- {
- return query.getSubQueryId();
- }
-
- @Override
- public MaterializedViewQuery withDataSource(DataSource dataSource)
- {
- return new MaterializedViewQuery(query.withDataSource(dataSource), optimizer);
- }
-
- @Override
- public String toString()
- {
- return "MaterializedViewQuery{" +
- "query=" + query +
- "}";
- }
-
- @Override
- public boolean equals(Object o)
- {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- MaterializedViewQuery other = (MaterializedViewQuery) o;
- return other.getQuery().equals(query);
- }
-
- @Override
- public int hashCode()
- {
- return Objects.hash(TYPE, query);
- }
-
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChest.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChest.java
deleted file mode 100644
index 3b59e290990..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChest.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Function;
-import com.google.inject.Inject;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryMetrics;
-import org.apache.druid.query.QueryPlus;
-import org.apache.druid.query.QueryRunner;
-import org.apache.druid.query.QueryToolChest;
-import org.apache.druid.query.QueryToolChestWarehouse;
-import org.apache.druid.query.aggregation.MetricManipulationFn;
-import org.apache.druid.query.context.ResponseContext;
-
-import java.util.Comparator;
-import java.util.function.BinaryOperator;
-
-public class MaterializedViewQueryQueryToolChest extends QueryToolChest
-{
- private final QueryToolChestWarehouse warehouse;
- private DataSourceOptimizer optimizer;
-
- @Inject
- public MaterializedViewQueryQueryToolChest(
- QueryToolChestWarehouse warehouse
- )
- {
- this.warehouse = warehouse;
- }
-
- @Override
- public QueryRunner mergeResults(QueryRunner runner)
- {
- return new QueryRunner()
- {
- @Override
- public Sequence run(QueryPlus queryPlus, ResponseContext responseContext)
- {
- Query realQuery = getRealQuery(queryPlus.getQuery());
- return warehouse.getToolChest(realQuery)
- .mergeResults(runner)
- .run(queryPlus.withQuery(realQuery), responseContext);
- }
- };
- }
-
- @Override
- public QueryRunner mergeResults(QueryRunner runner, boolean willMergeRunner)
- {
- return new QueryRunner()
- {
- @Override
- public Sequence run(QueryPlus queryPlus, ResponseContext responseContext)
- {
- Query realQuery = getRealQuery(queryPlus.getQuery());
- return warehouse.getToolChest(realQuery)
- .mergeResults(runner, willMergeRunner)
- .run(queryPlus.withQuery(realQuery), responseContext);
- }
- };
- }
-
- @Override
- public BinaryOperator createMergeFn(Query query)
- {
- final Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).createMergeFn(realQuery);
- }
-
- @Override
- public Comparator createResultComparator(Query query)
- {
- final Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).createResultComparator(realQuery);
- }
-
- @Override
- public QueryMetrics makeMetrics(Query query)
- {
- Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).makeMetrics(realQuery);
- }
-
- @Override
- public Function makePreComputeManipulatorFn(Query query, MetricManipulationFn fn)
- {
- Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).makePreComputeManipulatorFn(realQuery, fn);
- }
-
- @Override
- public Function makePostComputeManipulatorFn(Query query, MetricManipulationFn fn)
- {
- Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).makePostComputeManipulatorFn(realQuery, fn);
- }
-
- @Override
- public ObjectMapper decorateObjectMapper(final ObjectMapper objectMapper, final Query query)
- {
- Query realQuery = getRealQuery(query);
- return warehouse.getToolChest(realQuery).decorateObjectMapper(objectMapper, realQuery);
- }
-
- @Override
- public TypeReference getResultTypeReference()
- {
- return null;
- }
-
- @Override
- public QueryRunner preMergeQueryDecoration(final QueryRunner runner)
- {
- return new QueryRunner() {
- @Override
- public Sequence run(QueryPlus queryPlus, ResponseContext responseContext)
- {
- Query realQuery = getRealQuery(queryPlus.getQuery());
- QueryToolChest realQueryToolChest = warehouse.getToolChest(realQuery);
- QueryRunner realQueryRunner = realQueryToolChest.preMergeQueryDecoration(
- new MaterializedViewQueryRunner(runner, optimizer)
- );
- return realQueryRunner.run(queryPlus.withQuery(realQuery), responseContext);
- }
- };
- }
-
- public Query getRealQuery(Query query)
- {
- if (query instanceof MaterializedViewQuery) {
- optimizer = ((MaterializedViewQuery) query).getOptimizer();
- return ((MaterializedViewQuery) query).getQuery();
- }
- return query;
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryRunner.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryRunner.java
deleted file mode 100644
index 9c72808045c..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewQueryRunner.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.guava.MergeSequence;
-import org.apache.druid.java.util.common.guava.Sequence;
-import org.apache.druid.java.util.common.guava.Sequences;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryPlus;
-import org.apache.druid.query.QueryRunner;
-import org.apache.druid.query.context.ResponseContext;
-
-
-public class MaterializedViewQueryRunner<T> implements QueryRunner<T>
-{
- private final QueryRunner runner;
- private final DataSourceOptimizer optimizer;
-
- public MaterializedViewQueryRunner(QueryRunner queryRunner, DataSourceOptimizer optimizer)
- {
- this.runner = queryRunner;
- this.optimizer = optimizer;
- }
-
- @Override
- public Sequence<T> run(QueryPlus<T> queryPlus, ResponseContext responseContext)
- {
- Query query = queryPlus.getQuery();
- return new MergeSequence<>(
- query.getResultOrdering(),
- Sequences.simple(
- Lists.transform(
- optimizer.optimize(query),
- new Function<Query, Sequence<T>>()
- {
- @Override
- public Sequence<T> apply(Query query)
- {
- return runner.run(
- queryPlus.withQuery(query),
- responseContext
- );
- }
- }
- )
- )
- );
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewSelectionDruidModule.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewSelectionDruidModule.java
deleted file mode 100644
index f1e778b0cc8..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewSelectionDruidModule.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.databind.Module;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.google.common.collect.ImmutableList;
-import com.google.inject.Binder;
-import com.google.inject.Singleton;
-import org.apache.druid.discovery.NodeRole;
-import org.apache.druid.guice.DruidBinders;
-import org.apache.druid.guice.JsonConfigProvider;
-import org.apache.druid.guice.LifecycleModule;
-import org.apache.druid.guice.annotations.LoadScope;
-import org.apache.druid.initialization.DruidModule;
-import org.apache.druid.server.metrics.MetricsModule;
-
-import java.util.List;
-
-@LoadScope(roles = NodeRole.BROKER_JSON_NAME)
-public class MaterializedViewSelectionDruidModule implements DruidModule
-{
- @Override
- public List<? extends Module> getJacksonModules()
- {
- return ImmutableList.of(
- new SimpleModule(getClass().getSimpleName())
- .registerSubtypes(
- new NamedType(MaterializedViewQuery.class, MaterializedViewQuery.TYPE))
- );
- }
-
- @Override
- public void configure(Binder binder)
- {
- DruidBinders.queryToolChestBinder(binder)
- .addBinding(MaterializedViewQuery.class)
- .to(MaterializedViewQueryQueryToolChest.class);
- LifecycleModule.register(binder, DerivativeDataSourceManager.class);
- binder.bind(DataSourceOptimizer.class).in(Singleton.class);
- MetricsModule.register(binder, DataSourceOptimizerMonitor.class);
- JsonConfigProvider.bind(binder, "druid.manager.derivatives", MaterializedViewConfig.class);
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewUtils.java b/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewUtils.java
deleted file mode 100644
index 7c39b8fdc44..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/java/org/apache/druid/query/materializedview/MaterializedViewUtils.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.common.JodaUtils;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.FilteredAggregatorFactory;
-import org.apache.druid.query.dimension.DimensionSpec;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.topn.TopNQuery;
-import org.joda.time.Interval;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-public class MaterializedViewUtils
-{
- /**
- * extract all dimensions in query.
- * only support TopNQuery/TimeseriesQuery/GroupByQuery
- *
- * @param query
- * @return dimensions set in query
- */
- public static Set<String> getRequiredFields(Query query)
- {
- Set<String> dimsInFilter = null == query.getFilter() ? new HashSet<>() : query.getFilter().getRequiredColumns();
- Set<String> dimensions = new HashSet<>(dimsInFilter);
-
- if (query instanceof TopNQuery) {
- TopNQuery q = (TopNQuery) query;
- dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs()));
- dimensions.add(q.getDimensionSpec().getDimension());
- } else if (query instanceof TimeseriesQuery) {
- TimeseriesQuery q = (TimeseriesQuery) query;
- dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs()));
- } else if (query instanceof GroupByQuery) {
- GroupByQuery q = (GroupByQuery) query;
- dimensions.addAll(extractFieldsFromAggregations(q.getAggregatorSpecs()));
- for (DimensionSpec spec : q.getDimensions()) {
- String dim = spec.getDimension();
- dimensions.add(dim);
- }
- } else {
- throw new UnsupportedOperationException("Method getRequiredFields only supports TopNQuery/TimeseriesQuery/GroupByQuery");
- }
- return dimensions;
- }
-
- private static Set<String> extractFieldsFromAggregations(List<AggregatorFactory> aggs)
- {
- Set<String> ret = new HashSet<>();
- for (AggregatorFactory agg : aggs) {
- if (agg instanceof FilteredAggregatorFactory) {
- FilteredAggregatorFactory fagg = (FilteredAggregatorFactory) agg;
- ret.addAll(fagg.getFilter().getRequiredColumns());
- }
- ret.addAll(agg.requiredFields());
- }
- return ret;
- }
-
- /**
- * calculate the intervals which are covered by interval2, but not covered by interval1.
- * result intervals = interval2 - interval1 โฉ interval2
- * e.g.
- * a list of interval2: ["2018-04-01T00:00:00.000Z/2018-04-02T00:00:00.000Z",
- * "2018-04-03T00:00:00.000Z/2018-04-10T00:00:00.000Z"]
- * a list of interval1: ["2018-04-04T00:00:00.000Z/2018-04-06T00:00:00.000Z"]
- * the result list of intervals: ["2018-04-01T00:00:00.000Z/2018-04-02T00:00:00.000Z",
- * "2018-04-03T00:00:00.000Z/2018-04-04T00:00:00.000Z",
- * "2018-04-06T00:00:00.000Z/2018-04-10T00:00:00.000Z"]
- * If interval2 is empty, then return an empty list of interval.
- * @param interval2 list of intervals
- * @param interval1 list of intervals
- * @return list of intervals are covered by interval2, but not covered by interval1.
- */
- public static List<Interval> minus(List<Interval> interval2, List<Interval> interval1)
- {
- if (interval1.isEmpty() || interval2.isEmpty()) {
- return interval1;
- }
- Iterator<Interval> it1 = JodaUtils.condenseIntervals(interval1).iterator();
- Iterator<Interval> it2 = JodaUtils.condenseIntervals(interval2).iterator();
- List<Interval> remaining = new ArrayList<>();
- Interval currInterval1 = it1.next();
- Interval currInterval2 = it2.next();
- long start1 = currInterval1.getStartMillis();
- long end1 = currInterval1.getEndMillis();
- long start2 = currInterval2.getStartMillis();
- long end2 = currInterval2.getEndMillis();
- while (true) {
- if (start2 < start1 && end2 <= start1) {
- remaining.add(Intervals.utc(start2, end2));
- if (it2.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 < start1 && end2 > start1 && end2 < end1) {
- remaining.add(Intervals.utc(start2, start1));
- start1 = end2;
- if (it2.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 < start1 && end2 == end1) {
- remaining.add(Intervals.utc(start2, start1));
- if (it2.hasNext() && it1.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 < start1 && end2 > end1) {
- remaining.add(Intervals.utc(start2, start1));
- start2 = end1;
- if (it1.hasNext()) {
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- remaining.add(Intervals.utc(end1, end2));
- break;
- }
- }
- if (start2 == start1 && end2 >= start1 && end2 < end1) {
- start1 = end2;
- if (it2.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 == start1 && end2 > end1) {
- start2 = end1;
- if (it1.hasNext()) {
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- remaining.add(Intervals.utc(end1, end2));
- break;
- }
- }
- if (start2 > start1 && start2 < end1 && end2 < end1) {
- start1 = end2;
- if (it2.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 > start1 && start2 < end1 && end2 > end1) {
- start2 = end1;
- if (it1.hasNext()) {
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- remaining.add(Intervals.utc(end1, end2));
- break;
- }
- }
- if (start2 >= start1 && start2 <= end1 && end2 == end1) {
- if (it2.hasNext() && it1.hasNext()) {
- currInterval2 = it2.next();
- start2 = currInterval2.getStartMillis();
- end2 = currInterval2.getEndMillis();
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- break;
- }
- }
- if (start2 >= end1 && end2 > end1) {
- if (it1.hasNext()) {
- currInterval1 = it1.next();
- start1 = currInterval1.getStartMillis();
- end1 = currInterval1.getEndMillis();
- } else {
- remaining.add(Intervals.utc(start2, end2));
- break;
- }
- }
- }
-
- while (it2.hasNext()) {
- remaining.add(Intervals.of(it2.next().toString()));
- }
- return remaining;
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule b/extensions-contrib/materialized-view-selection/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
deleted file mode 100644
index a0aa8be4c8c..00000000000
--- a/extensions-contrib/materialized-view-selection/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.druid.query.materializedview.MaterializedViewSelectionDruidModule
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java
deleted file mode 100644
index 4e42beb0d28..00000000000
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DataSourceOptimizerMonitorTest.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import org.apache.druid.java.util.metrics.StubServiceEmitter;
-import org.easymock.EasyMock;
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Map;
-
-public class DataSourceOptimizerMonitorTest
-{
-
- @Test
- public void testDoMonitor()
- {
- final DataSourceOptimizerStats wikiStats
- = new DataSourceOptimizerStats("wiki", 100, 200, 30, Collections.emptyMap(), Collections.emptyMap());
- final DataSourceOptimizerStats koalaStats
- = new DataSourceOptimizerStats("koala", 35, 62, 13, Collections.emptyMap(), Collections.emptyMap());
-
- DataSourceOptimizer optimizer = EasyMock.createMock(DataSourceOptimizer.class);
- EasyMock.expect(optimizer.getAndResetStats()).andReturn(
- Arrays.asList(wikiStats, koalaStats)
- ).anyTimes();
- EasyMock.replay(optimizer);
-
- DataSourceOptimizerMonitor monitor = new DataSourceOptimizerMonitor(optimizer);
- StubServiceEmitter emitter = new StubServiceEmitter("test", "localhost");
- monitor.doMonitor(emitter);
-
- // Verify metrics emitted for all the datasources
- Map<String, Object> wikiFilter = Collections.singletonMap("dataSource", "wiki");
- emitter.verifyValue("/materialized/view/query/totalNum", wikiFilter, wikiStats.getTotalcount());
- emitter.verifyValue("/materialized/view/query/hits", wikiFilter, wikiStats.getHitcount());
- emitter.verifyValue("/materialized/view/query/hitRate", wikiFilter, wikiStats.getHitRate());
- emitter.verifyValue("/materialized/view/select/avgCostMS", wikiFilter, wikiStats.getOptimizerCost());
-
- Map<String, Object> koalaFilter = Collections.singletonMap("dataSource", "koala");
- emitter.verifyValue("/materialized/view/query/totalNum", koalaFilter, koalaStats.getTotalcount());
- emitter.verifyValue("/materialized/view/query/hits", koalaFilter, koalaStats.getHitcount());
- emitter.verifyValue("/materialized/view/query/hitRate", koalaFilter, koalaStats.getHitRate());
- emitter.verifyValue("/materialized/view/select/avgCostMS", koalaFilter, koalaStats.getOptimizerCost());
-
-
- EasyMock.verify(optimizer);
- }
-
-}
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java
deleted file mode 100644
index 9cd368293f9..00000000000
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.fasterxml.jackson.dataformat.smile.SmileGenerator;
-import com.google.common.base.Predicates;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.druid.client.BatchServerInventoryView;
-import org.apache.druid.client.BrokerSegmentWatcherConfig;
-import org.apache.druid.client.BrokerServerView;
-import org.apache.druid.client.BrokerViewOfCoordinatorConfig;
-import org.apache.druid.client.DirectDruidClientFactory;
-import org.apache.druid.client.DruidServer;
-import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy;
-import org.apache.druid.client.selector.RandomServerSelectorStrategy;
-import org.apache.druid.curator.CuratorTestBase;
-import org.apache.druid.indexing.materializedview.DerivativeDataSourceMetadata;
-import org.apache.druid.jackson.DefaultObjectMapper;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.metadata.IndexerSQLMetadataStorageCoordinator;
-import org.apache.druid.metadata.TestDerbyConnector;
-import org.apache.druid.metadata.segment.SqlSegmentMetadataTransactionFactory;
-import org.apache.druid.metadata.segment.cache.NoopSegmentMetadataCache;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryRunnerFactoryConglomerate;
-import org.apache.druid.query.QueryRunnerTestHelper;
-import org.apache.druid.query.QueryWatcher;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
-import org.apache.druid.query.topn.TopNQuery;
-import org.apache.druid.query.topn.TopNQueryBuilder;
-import org.apache.druid.segment.TestHelper;
-import org.apache.druid.segment.metadata.CentralizedDatasourceSchemaConfig;
-import org.apache.druid.segment.metadata.HeapMemoryIndexingStateStorage;
-import org.apache.druid.segment.metadata.SegmentSchemaManager;
-import org.apache.druid.segment.realtime.appenderator.SegmentSchemas;
-import org.apache.druid.server.coordination.DruidServerMetadata;
-import org.apache.druid.server.coordination.ServerType;
-import org.apache.druid.server.coordination.TestCoordinatorClient;
-import org.apache.druid.server.coordinator.simulate.TestDruidLeaderSelector;
-import org.apache.druid.server.initialization.ZkPathsConfig;
-import org.apache.druid.server.metrics.NoopServiceEmitter;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.NoneShardSpec;
-import org.easymock.EasyMock;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Executor;
-import java.util.concurrent.TimeUnit;
-
-public class DatasourceOptimizerTest extends CuratorTestBase
-{
- @Rule
- public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule();
- private DerivativeDataSourceManager derivativesManager;
- private DruidServer druidServer;
- private ObjectMapper jsonMapper;
- private ZkPathsConfig zkPathsConfig;
- private DataSourceOptimizer optimizer;
- private IndexerSQLMetadataStorageCoordinator metadataStorageCoordinator;
- private BatchServerInventoryView baseView;
- private BrokerServerView brokerServerView;
- private SegmentSchemaManager segmentSchemaManager;
-
- @Before
- public void setUp() throws Exception
- {
- TestDerbyConnector derbyConnector = derbyConnectorRule.getConnector();
- derbyConnector.createDataSourceTable();
- derbyConnector.createSegmentSchemasTable();
- derbyConnector.createSegmentTable();
- MaterializedViewConfig viewConfig = new MaterializedViewConfig();
- jsonMapper = TestHelper.makeJsonMapper();
- jsonMapper.registerSubtypes(new NamedType(DerivativeDataSourceMetadata.class, "view"));
- metadataStorageCoordinator = EasyMock.createMock(IndexerSQLMetadataStorageCoordinator.class);
- derivativesManager = new DerivativeDataSourceManager(
- viewConfig,
- derbyConnectorRule.metadataTablesConfigSupplier(),
- jsonMapper,
- derbyConnector
- );
- segmentSchemaManager = new SegmentSchemaManager(
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- jsonMapper,
- derbyConnector
- );
-
- metadataStorageCoordinator = new IndexerSQLMetadataStorageCoordinator(
- new SqlSegmentMetadataTransactionFactory(
- jsonMapper,
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- derbyConnector,
- new TestDruidLeaderSelector(),
- NoopSegmentMetadataCache.instance(),
- NoopServiceEmitter.instance()
- ),
- jsonMapper,
- derbyConnectorRule.metadataTablesConfigSupplier().get(),
- derbyConnector,
- segmentSchemaManager,
- CentralizedDatasourceSchemaConfig.create(),
- new HeapMemoryIndexingStateStorage()
- );
-
- setupServerAndCurator();
- curator.start();
- curator.blockUntilConnected();
-
- zkPathsConfig = new ZkPathsConfig();
- setupViews();
-
- druidServer = new DruidServer(
- "localhost:1234",
- "localhost:1234",
- null,
- 10000000L,
- null,
- ServerType.HISTORICAL,
- "default_tier",
- 0
- );
- setupZNodeForServer(druidServer, new ZkPathsConfig(), jsonMapper);
- optimizer = new DataSourceOptimizer(brokerServerView);
- }
-
- @After
- public void tearDown() throws IOException
- {
- baseView.stop();
- tearDownServerAndCurator();
- }
-
- @Test(timeout = 60_000L)
- public void testOptimize() throws InterruptedException
- {
- // insert datasource metadata
- String dataSource = "derivative";
- String baseDataSource = "base";
- Set<String> dims = Sets.newHashSet("dim1", "dim2", "dim3");
- Set<String> metrics = Sets.newHashSet("cost");
- DerivativeDataSourceMetadata metadata = new DerivativeDataSourceMetadata(baseDataSource, dims, metrics);
- metadataStorageCoordinator.insertDataSourceMetadata(dataSource, metadata);
- // insert base datasource segments
- List<Boolean> baseResult = Lists.transform(
- ImmutableList.of(
- "2011-04-01/2011-04-02",
- "2011-04-02/2011-04-03",
- "2011-04-03/2011-04-04",
- "2011-04-04/2011-04-05",
- "2011-04-05/2011-04-06"
- ),
- interval -> {
- final DataSegment segment = createDataSegment(
- "base",
- interval,
- "v1",
- Lists.newArrayList("dim1", "dim2", "dim3", "dim4"),
- 1024 * 1024
- );
- metadataStorageCoordinator.commitSegments(Sets.newHashSet(segment), null);
- announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper);
- return true;
- }
- );
- // insert derivative segments
- List<Boolean> derivativeResult = Lists.transform(
- ImmutableList.of(
- "2011-04-01/2011-04-02",
- "2011-04-02/2011-04-03",
- "2011-04-03/2011-04-04"
- ),
- interval -> {
- final DataSegment segment = createDataSegment(
- "derivative",
- interval,
- "v1",
- Lists.newArrayList("dim1", "dim2", "dim3"),
- 1024
- );
- metadataStorageCoordinator.commitSegments(Sets.newHashSet(segment), null);
- announceSegmentForServer(druidServer, segment, zkPathsConfig, jsonMapper);
- return true;
- }
- );
- Assert.assertFalse(baseResult.contains(false));
- Assert.assertFalse(derivativeResult.contains(false));
- derivativesManager.start();
- while (DerivativeDataSourceManager.getAllDerivatives().isEmpty()) {
- TimeUnit.SECONDS.sleep(1L);
- }
- // build user query
- TopNQuery userQuery = new TopNQueryBuilder()
- .dataSource("base")
- .granularity(QueryRunnerTestHelper.ALL_GRAN)
- .dimension("dim1")
- .metric("cost")
- .threshold(4)
- .intervals("2011-04-01/2011-04-06")
- .aggregators(new LongSumAggregatorFactory("cost", "cost"))
- .build();
-
- List<Query> expectedQueryAfterOptimizing = Lists.newArrayList(
- new TopNQueryBuilder()
- .dataSource("derivative")
- .granularity(QueryRunnerTestHelper.ALL_GRAN)
- .dimension("dim1")
- .metric("cost")
- .threshold(4)
- .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2011-04-01/2011-04-04"))))
- .aggregators(new LongSumAggregatorFactory("cost", "cost"))
- .build(),
- new TopNQueryBuilder()
- .dataSource("base")
- .granularity(QueryRunnerTestHelper.ALL_GRAN)
- .dimension("dim1")
- .metric("cost")
- .threshold(4)
- .intervals(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2011-04-04/2011-04-06"))))
- .aggregators(new LongSumAggregatorFactory("cost", "cost"))
- .build()
- );
- Assert.assertEquals(expectedQueryAfterOptimizing, optimizer.optimize(userQuery));
- derivativesManager.stop();
- }
-
- private DataSegment createDataSegment(String name, String intervalStr, String version, List<String> dims, long size)
- {
- return DataSegment.builder()
- .dataSource(name)
- .interval(Intervals.of(intervalStr))
- .loadSpec(
- ImmutableMap.of(
- "type",
- "local",
- "path",
- "somewhere"
- )
- )
- .version(version)
- .dimensions(dims)
- .metrics(ImmutableList.of("cost"))
- .shardSpec(NoneShardSpec.instance())
- .binaryVersion(9)
- .size(size)
- .build();
- }
-
- private void setupViews() throws Exception
- {
- baseView = new BatchServerInventoryView(zkPathsConfig, curator, jsonMapper, Predicates.alwaysTrue(), "test")
- {
- @Override
- public void registerSegmentCallback(Executor exec, final SegmentCallback callback)
- {
- super.registerSegmentCallback(
- exec,
- new SegmentCallback()
- {
- @Override
- public CallbackAction segmentAdded(DruidServerMetadata server, DataSegment segment)
- {
- return callback.segmentAdded(server, segment);
- }
-
- @Override
- public CallbackAction segmentRemoved(DruidServerMetadata server, DataSegment segment)
- {
- return callback.segmentRemoved(server, segment);
- }
-
- @Override
- public CallbackAction segmentViewInitialized()
- {
- return callback.segmentViewInitialized();
- }
-
- @Override
- public CallbackAction segmentSchemasAnnounced(SegmentSchemas segmentSchemas)
- {
- return CallbackAction.CONTINUE;
- }
- }
- );
- }
- };
-
- DirectDruidClientFactory druidClientFactory = new DirectDruidClientFactory(
- new NoopServiceEmitter(),
- EasyMock.createMock(QueryRunnerFactoryConglomerate.class),
- EasyMock.createMock(QueryWatcher.class),
- getSmileMapper(),
- EasyMock.createMock(HttpClient.class)
- );
-
- BrokerViewOfCoordinatorConfig filter = new BrokerViewOfCoordinatorConfig(new TestCoordinatorClient());
- filter.start();
- brokerServerView = new BrokerServerView(
- druidClientFactory,
- baseView,
- new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()),
- new HighestPriorityTierSelectorStrategy(new RandomServerSelectorStrategy()),
- new NoopServiceEmitter(),
- new BrokerSegmentWatcherConfig(),
- filter
- );
- baseView.start();
- }
-
- private ObjectMapper getSmileMapper()
- {
- final SmileFactory smileFactory = new SmileFactory();
- smileFactory.configure(SmileGenerator.Feature.ENCODE_BINARY_AS_7BIT, false);
- smileFactory.delegateToTextual(true);
- final ObjectMapper retVal = new DefaultObjectMapper(smileFactory, "broker");
- retVal.getFactory().setCodec(retVal);
- return retVal;
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java
deleted file mode 100644
index ba534070da5..00000000000
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Function;
-import com.google.common.collect.ImmutableMap;
-import org.apache.druid.data.input.MapBasedRow;
-import org.apache.druid.jackson.DefaultObjectMapper;
-import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.query.Druids;
-import org.apache.druid.query.MapQueryToolChestWarehouse;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryRunnerTestHelper;
-import org.apache.druid.query.QueryToolChest;
-import org.apache.druid.query.Result;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.aggregation.MetricManipulationFn;
-import org.apache.druid.query.dimension.DefaultDimensionSpec;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.groupby.GroupByQueryConfig;
-import org.apache.druid.query.groupby.GroupByQueryQueryToolChest;
-import org.apache.druid.query.groupby.GroupByQueryRunnerTestHelper;
-import org.apache.druid.query.groupby.ResultRow;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest;
-import org.apache.druid.query.timeseries.TimeseriesResultValue;
-import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-public class MaterializedViewQueryQueryToolChestTest extends InitializedNullHandlingTest
-{
- private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
-
- @Test
- public void testMakePostComputeManipulatorFn()
- {
- TimeseriesQuery realQuery = Druids.newTimeseriesQueryBuilder()
- .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .granularity(QueryRunnerTestHelper.DAY_GRAN)
- .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
- .aggregators(QueryRunnerTestHelper.ROWS_COUNT)
- .descending(true)
- .build();
- MaterializedViewQuery materializedViewQuery = new MaterializedViewQuery(realQuery, null);
-
- QueryToolChest materializedViewQueryQueryToolChest =
- new MaterializedViewQueryQueryToolChest(new MapQueryToolChestWarehouse(
- ImmutableMap.<Class<? extends Query>, QueryToolChest>builder()
- .put(TimeseriesQuery.class, new TimeseriesQueryQueryToolChest())
- .build()
- ));
-
- Function postFn =
- materializedViewQueryQueryToolChest.makePostComputeManipulatorFn(
- materializedViewQuery,
- new MetricManipulationFn() {
- @Override
- public Object manipulate(AggregatorFactory factory, Object object)
- {
- return "metricvalue1";
- }
- });
-
-
-
- Result<TimeseriesResultValue> result = new Result<>(
- DateTimes.nowUtc(),
- new TimeseriesResultValue(ImmutableMap.of("dim1", "dimvalue1"))
- );
-
- Result<TimeseriesResultValue> postResult = (Result<TimeseriesResultValue>) postFn.apply(result);
- Map<String, Object> postResultMap = postResult.getValue().getBaseObject();
-
- Assert.assertEquals(postResult.getTimestamp(), result.getTimestamp());
- Assert.assertEquals(postResultMap.size(), 2);
- Assert.assertEquals(postResultMap.get(QueryRunnerTestHelper.ROWS_COUNT.getName()), "metricvalue1");
- Assert.assertEquals(postResultMap.get("dim1"), "dimvalue1");
- }
-
- @Test
- public void testDecorateObjectMapper() throws IOException
- {
- GroupByQuery realQuery = GroupByQuery.builder()
- .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
- .setDimensions(new DefaultDimensionSpec("quality", "alias"))
- .setAggregatorSpecs(
- QueryRunnerTestHelper.ROWS_COUNT,
- new LongSumAggregatorFactory("idx", "index")
- )
- .setGranularity(QueryRunnerTestHelper.DAY_GRAN)
- .setContext(ImmutableMap.of(GroupByQueryConfig.CTX_KEY_ARRAY_RESULT_ROWS, false))
- .build();
-
- QueryToolChest queryToolChest =
- new MaterializedViewQueryQueryToolChest(new MapQueryToolChestWarehouse(
- ImmutableMap.<Class<? extends Query>, QueryToolChest>builder()
- .put(GroupByQuery.class, new GroupByQueryQueryToolChest(null, null))
- .build()
- ));
-
- ObjectMapper objectMapper = queryToolChest.decorateObjectMapper(JSON_MAPPER, realQuery);
-
- List<ResultRow> results = Arrays.asList(
- GroupByQueryRunnerTestHelper.createExpectedRow(
- realQuery,
- "2011-04-01",
- "alias",
- "automotive",
- "rows",
- 1L,
- "idx",
- 135L
-
- ),
- GroupByQueryRunnerTestHelper.createExpectedRow(
- realQuery,
- "2011-04-01",
- "alias",
- "business",
- "rows",
- 1L,
- "idx",
- 118L
- )
- );
- List<MapBasedRow> expectedResults = results.stream()
- .map(resultRow -> resultRow.toMapBasedRow(realQuery))
- .collect(Collectors.toList());
-
- Assert.assertEquals(
- "decorate-object-mapper",
- JSON_MAPPER.writerFor(new TypeReference<List<MapBasedRow>>(){}).writeValueAsString(expectedResults),
- objectMapper.writeValueAsString(results)
- );
- }
-
- @Test
- public void testDecorateObjectMapperMaterializedViewQuery() throws IOException
- {
- GroupByQuery realQuery = GroupByQuery.builder()
- .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
- .setDimensions(new DefaultDimensionSpec("quality", "alias"))
- .setAggregatorSpecs(
- QueryRunnerTestHelper.ROWS_COUNT,
- new LongSumAggregatorFactory("idx", "index")
- )
- .setGranularity(QueryRunnerTestHelper.DAY_GRAN)
- .setContext(ImmutableMap.of(GroupByQueryConfig.CTX_KEY_ARRAY_RESULT_ROWS, false))
- .build();
- MaterializedViewQuery materializedViewQuery = new MaterializedViewQuery(realQuery, null);
-
- QueryToolChest materializedViewQueryQueryToolChest =
- new MaterializedViewQueryQueryToolChest(new MapQueryToolChestWarehouse(
- ImmutableMap.<Class<? extends Query>, QueryToolChest>builder()
- .put(GroupByQuery.class, new GroupByQueryQueryToolChest(null, null))
- .build()
- ));
-
- ObjectMapper objectMapper = materializedViewQueryQueryToolChest.decorateObjectMapper(JSON_MAPPER, materializedViewQuery);
-
- List<ResultRow> results = Arrays.asList(
- GroupByQueryRunnerTestHelper.createExpectedRow(
- realQuery,
- "2011-04-01",
- "alias",
- "automotive",
- "rows",
- 1L,
- "idx",
- 135L
-
- ),
- GroupByQueryRunnerTestHelper.createExpectedRow(
- realQuery,
- "2011-04-01",
- "alias",
- "business",
- "rows",
- 1L,
- "idx",
- 118L
- )
- );
- List<MapBasedRow> expectedResults = results.stream()
- .map(resultRow -> resultRow.toMapBasedRow(realQuery))
- .collect(Collectors.toList());
-
- Assert.assertEquals(
- "decorate-object-mapper",
- JSON_MAPPER.writerFor(new TypeReference<List<MapBasedRow>>(){}).writeValueAsString(expectedResults),
- objectMapper.writeValueAsString(results)
- );
- }
-
- @Test
- public void testGetRealQuery()
- {
- GroupByQuery realQuery = GroupByQuery.builder()
- .setDataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD)
- .setDimensions(new DefaultDimensionSpec("quality", "alias"))
- .setAggregatorSpecs(
- QueryRunnerTestHelper.ROWS_COUNT,
- new LongSumAggregatorFactory("idx", "index")
- )
- .setGranularity(QueryRunnerTestHelper.DAY_GRAN)
- .setContext(ImmutableMap.of(GroupByQueryConfig.CTX_KEY_ARRAY_RESULT_ROWS, false))
- .build();
- MaterializedViewQuery materializedViewQuery = new MaterializedViewQuery(realQuery, null);
-
- MaterializedViewQueryQueryToolChest materializedViewQueryQueryToolChest =
- new MaterializedViewQueryQueryToolChest(new MapQueryToolChestWarehouse(
- ImmutableMap.<Class<? extends Query>, QueryToolChest>builder()
- .put(GroupByQuery.class, new GroupByQueryQueryToolChest(null, null))
- .build()
- ));
-
- Assert.assertEquals(realQuery, materializedViewQueryQueryToolChest.getRealQuery(materializedViewQuery));
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java
deleted file mode 100644
index ad9913eca55..00000000000
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.databind.InjectableValues;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.druid.java.util.common.HumanReadableBytes;
-import org.apache.druid.math.expr.ExprMacroTable;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryRunnerTestHelper;
-import org.apache.druid.query.TableDataSource;
-import org.apache.druid.query.aggregation.DoubleMaxAggregatorFactory;
-import org.apache.druid.query.aggregation.DoubleMinAggregatorFactory;
-import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable;
-import org.apache.druid.query.topn.TopNQuery;
-import org.apache.druid.query.topn.TopNQueryBuilder;
-import org.apache.druid.segment.TestHelper;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.io.IOException;
-
-public class MaterializedViewQueryTest
-{
- private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper();
- private DataSourceOptimizer optimizer;
-
- @Before
- public void setUp()
- {
- JSON_MAPPER.registerSubtypes(new NamedType(MaterializedViewQuery.class, MaterializedViewQuery.TYPE));
- optimizer = EasyMock.createMock(DataSourceOptimizer.class);
- JSON_MAPPER.setInjectableValues(
- new InjectableValues.Std()
- .addValue(ExprMacroTable.class.getName(), LookupEnabledTestExprMacroTable.INSTANCE)
- .addValue(DataSourceOptimizer.class, optimizer)
- );
- }
-
- @Test
- public void testQuerySerialization() throws IOException
- {
- TopNQuery topNQuery = new TopNQueryBuilder()
- .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .granularity(QueryRunnerTestHelper.ALL_GRAN)
- .dimension(QueryRunnerTestHelper.MARKET_DIMENSION)
- .metric(QueryRunnerTestHelper.INDEX_METRIC)
- .threshold(4)
- .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
- .aggregators(
- Lists.newArrayList(
- Iterables.concat(
- QueryRunnerTestHelper.COMMON_DOUBLE_AGGREGATORS,
- Lists.newArrayList(
- new DoubleMaxAggregatorFactory("maxIndex", "index"),
- new DoubleMinAggregatorFactory("minIndex", "index")
- )
- )
- )
- )
- .postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT)
- .build();
- MaterializedViewQuery query = new MaterializedViewQuery(topNQuery, optimizer);
- String json = JSON_MAPPER.writeValueAsString(query);
- Query serdeQuery = JSON_MAPPER.readValue(json, Query.class);
- Assert.assertEquals(query, serdeQuery);
- Assert.assertEquals(new TableDataSource(QueryRunnerTestHelper.DATA_SOURCE), query.getDataSource());
- Assert.assertEquals(QueryRunnerTestHelper.ALL_GRAN, query.getGranularity());
- Assert.assertEquals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC.getIntervals(), query.getIntervals());
- }
-
- @Test
- public void testGetContextHumanReadableBytes()
- {
- TopNQuery topNQuery = new TopNQueryBuilder()
- .dataSource(QueryRunnerTestHelper.DATA_SOURCE)
- .granularity(QueryRunnerTestHelper.ALL_GRAN)
- .dimension(QueryRunnerTestHelper.MARKET_DIMENSION)
- .metric(QueryRunnerTestHelper.INDEX_METRIC)
- .threshold(4)
- .intervals(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC)
- .aggregators(
- Lists.newArrayList(
- Iterables.concat(
- QueryRunnerTestHelper.COMMON_DOUBLE_AGGREGATORS,
- Lists.newArrayList(
- new DoubleMaxAggregatorFactory("maxIndex", "index"),
- new DoubleMinAggregatorFactory("minIndex", "index")
- )
- )
- )
- )
- .context(
- ImmutableMap.of(
- "maxOnDiskStorage", "20M"
- )
- )
- .postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT)
- .build();
- MaterializedViewQuery query = new MaterializedViewQuery(topNQuery, optimizer);
- Assert.assertEquals(20_000_000, query.context().getHumanReadableBytes("maxOnDiskStorage", HumanReadableBytes.ZERO).getBytes());
- }
-}
diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewUtilsTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewUtilsTest.java
deleted file mode 100644
index fe00a2ce318..00000000000
--- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewUtilsTest.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.query.materializedview;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import org.apache.druid.java.util.common.Intervals;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.topn.TopNQuery;
-import org.apache.druid.segment.TestHelper;
-import org.apache.druid.testing.InitializedNullHandlingTest;
-import org.joda.time.Interval;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.List;
-import java.util.Set;
-
-public class MaterializedViewUtilsTest extends InitializedNullHandlingTest
-{
- private static ObjectMapper jsonMapper = TestHelper.makeJsonMapper();
-
- @Test
- public void testGetRequiredFieldsFromGroupByQuery() throws Exception
- {
- String queryStr = "{\n" +
- " \"queryType\": \"groupBy\",\n" +
- " \"dataSource\": \"sample_datasource\",\n" +
- " \"granularity\": \"day\",\n" +
- " \"dimensions\": [\"country\", \"device\"],\n" +
- " \"limitSpec\": { \"type\": \"default\", \"limit\": 5000, \"columns\": [\"country\", \"data_transfer\"] },\n" +
- " \"filter\": {\n" +
- " \"type\": \"and\",\n" +
- " \"fields\": [\n" +
- " { \"type\": \"selector\", \"dimension\": \"carrier\", \"value\": \"AT&T\" },\n" +
- " { \"type\": \"or\", \n" +
- " \"fields\": [\n" +
- " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Apple\" },\n" +
- " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Samsung\" }\n" +
- " ]\n" +
- " }\n" +
- " ]\n" +
- " },\n" +
- " \"aggregations\": [\n" +
- " { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" },\n" +
- " { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" }\n" +
- " ],\n" +
- " \"postAggregations\": [\n" +
- " { \"type\": \"arithmetic\",\n" +
- " \"name\": \"avg_usage\",\n" +
- " \"fn\": \"/\",\n" +
- " \"fields\": [\n" +
- " { \"type\": \"fieldAccess\", \"fieldName\": \"data_transfer\" },\n" +
- " { \"type\": \"fieldAccess\", \"fieldName\": \"total_usage\" }\n" +
- " ]\n" +
- " }\n" +
- " ],\n" +
- " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ],\n" +
- " \"having\": {\n" +
- " \"type\": \"greaterThan\",\n" +
- " \"aggregation\": \"total_usage\",\n" +
- " \"value\": 100\n" +
- " }\n" +
- "}";
- GroupByQuery query = jsonMapper.readValue(queryStr, GroupByQuery.class);
- Set<String> fields = MaterializedViewUtils.getRequiredFields(query);
- Assert.assertEquals(
- Sets.newHashSet("country", "device", "carrier", "make", "user_count", "data_transfer"),
- fields
- );
- }
-
- @Test
- public void testGetRequiredFieldsFromTopNQuery() throws Exception
- {
- String queryStr = "{\n" +
- " \"queryType\": \"topN\",\n" +
- " \"dataSource\": \"sample_data\",\n" +
- " \"dimension\": \"sample_dim\",\n" +
- " \"threshold\": 5,\n" +
- " \"metric\": \"count\",\n" +
- " \"granularity\": \"all\",\n" +
- " \"filter\": {\n" +
- " \"type\": \"and\",\n" +
- " \"fields\": [\n" +
- " {\n" +
- " \"type\": \"selector\",\n" +
- " \"dimension\": \"dim1\",\n" +
- " \"value\": \"some_value\"\n" +
- " },\n" +
- " {\n" +
- " \"type\": \"selector\",\n" +
- " \"dimension\": \"dim2\",\n" +
- " \"value\": \"some_other_val\"\n" +
- " }\n" +
- " ]\n" +
- " },\n" +
- " \"aggregations\": [\n" +
- " {\n" +
- " \"type\": \"longSum\",\n" +
- " \"name\": \"count\",\n" +
- " \"fieldName\": \"count\"\n" +
- " },\n" +
- " {\n" +
- " \"type\": \"doubleSum\",\n" +
- " \"name\": \"some_metric\",\n" +
- " \"fieldName\": \"some_metric\"\n" +
- " }\n" +
- " ],\n" +
- " \"postAggregations\": [\n" +
- " {\n" +
- " \"type\": \"arithmetic\",\n" +
- " \"name\": \"average\",\n" +
- " \"fn\": \"/\",\n" +
- " \"fields\": [\n" +
- " {\n" +
- " \"type\": \"fieldAccess\",\n" +
- " \"name\": \"some_metric\",\n" +
- " \"fieldName\": \"some_metric\"\n" +
- " },\n" +
- " {\n" +
- " \"type\": \"fieldAccess\",\n" +
- " \"name\": \"count\",\n" +
- " \"fieldName\": \"count\"\n" +
- " }\n" +
- " ]\n" +
- " }\n" +
- " ],\n" +
- " \"intervals\": [\n" +
- " \"2013-08-31T00:00:00.000/2013-09-03T00:00:00.000\"\n" +
- " ]\n" +
- "}";
- TopNQuery query = jsonMapper.readValue(queryStr, TopNQuery.class);
- Set<String> fields = MaterializedViewUtils.getRequiredFields(query);
- Assert.assertEquals(
- Sets.newHashSet("sample_dim", "dim1", "dim2", "count", "some_metric"),
- fields
- );
- }
-
- @Test
- public void testGetRequiredFieldsFromTimeseriesQuery() throws Exception
- {
- String queryStr = "{\n" +
- " \"queryType\": \"timeseries\",\n" +
- " \"dataSource\": \"sample_datasource\",\n" +
- " \"granularity\": \"day\",\n" +
- " \"descending\": \"true\",\n" +
- " \"filter\": {\n" +
- " \"type\": \"and\",\n" +
- " \"fields\": [\n" +
- " { \"type\": \"selector\", \"dimension\": \"sample_dimension1\", \"value\": \"sample_value1\" },\n" +
- " { \"type\": \"or\",\n" +
- " \"fields\": [\n" +
- " { \"type\": \"selector\", \"dimension\": \"sample_dimension2\", \"value\": \"sample_value2\" },\n" +
- " { \"type\": \"selector\", \"dimension\": \"sample_dimension3\", \"value\": \"sample_value3\" }\n" +
- " ]\n" +
- " }\n" +
- " ]\n" +
- " },\n" +
- " \"aggregations\": [\n" +
- " { \"type\": \"longSum\", \"name\": \"sample_name1\", \"fieldName\": \"sample_fieldName1\" },\n" +
- " { \"type\": \"doubleSum\", \"name\": \"sample_name2\", \"fieldName\": \"sample_fieldName2\" }\n" +
- " ],\n" +
- " \"postAggregations\": [\n" +
- " { \"type\": \"arithmetic\",\n" +
- " \"name\": \"sample_divide\",\n" +
- " \"fn\": \"/\",\n" +
- " \"fields\": [\n" +
- " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name1\", \"fieldName\": \"sample_name1\" },\n" +
- " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name2\", \"fieldName\": \"sample_name2\" }\n" +
- " ]\n" +
- " }\n" +
- " ],\n" +
- " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ]\n" +
- "}";
- TimeseriesQuery query = jsonMapper.readValue(queryStr, TimeseriesQuery.class);
- Set<String> fields = MaterializedViewUtils.getRequiredFields(query);
- Assert.assertEquals(
- Sets.newHashSet("sample_dimension1", "sample_dimension2", "sample_dimension3", "sample_fieldName1",
- "sample_fieldName2"),
- fields
- );
- }
-
- @Test
- public void testIntervalMinus()
- {
- List<Interval> intervalList1 = Lists.newArrayList(
- Intervals.of("2012-01-02T00:00:00.000/2012-01-03T00:00:00.000"),
- Intervals.of("2012-01-08T00:00:00.000/2012-01-10T00:00:00.000"),
- Intervals.of("2012-01-16T00:00:00.000/2012-01-17T00:00:00.000")
- );
- List<Interval> intervalList2 = Lists.newArrayList(
- Intervals.of("2012-01-01T00:00:00.000/2012-01-04T00:00:00.000"),
- Intervals.of("2012-01-05T00:00:00.000/2012-01-10T00:00:00.000"),
- Intervals.of("2012-01-16T00:00:00.000/2012-01-18T00:00:00.000"),
- Intervals.of("2012-01-19T00:00:00.000/2012-01-20T00:00:00.000")
- );
-
- List<Interval> result = MaterializedViewUtils.minus(intervalList2, intervalList1);
- Assert.assertEquals(
- Lists.newArrayList(
- Intervals.of("2012-01-01T00:00:00.000/2012-01-02T00:00:00.000"),
- Intervals.of("2012-01-03T00:00:00.000/2012-01-04T00:00:00.000"),
- Intervals.of("2012-01-05T00:00:00.000/2012-01-08T00:00:00.000"),
- Intervals.of("2012-01-17T00:00:00.000/2012-01-18T00:00:00.000"),
- Intervals.of("2012-01-19T00:00:00.000/2012-01-20T00:00:00.000")
- ),
- result
- );
- }
-}
diff --git a/extensions-contrib/thrift-extensions/example/books.json b/extensions-contrib/thrift-extensions/example/books.json
deleted file mode 100644
index 6f1dc54c00b..00000000000
--- a/extensions-contrib/thrift-extensions/example/books.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
- "type" : "index_hadoop",
- "spec" : {
- "dataSchema" : {
- "dataSource" : "test",
- "parser" : {
- "type" : "thrift",
- "jarPath" : "example/book.jar",
- "thriftClass": "org.apache.druid.data.input.thrift.Book",
- "protocol" : "compact",
- "parseSpec" : {
- "format" : "json",
- "timestampSpec" : {
- "column" : "date",
- "format" : "auto"
- },
- "flattenSpec" : {
- "useFieldDiscovery" : true,
- "fields" : [ {
- "type" : "path",
- "name" : "lastName",
- "expr" : "$.author.lastName"
- }, "title" ]
- },
- "dimensionsSpec" : {
- "dimensions" : [ "title", "lastName" ]
- }
- }
- },
- "metricsSpec" : [ {
- "type" : "count",
- "name" : "count"
- }, {
- "type" : "doubleSum",
- "name" : "cost",
- "fieldName" : "price"
- } ],
- "granularitySpec" : {
- "type" : "uniform",
- "segmentGranularity" : "DAY",
- "queryGranularity" : "DAY",
- "intervals" : [ "2015-09-01/2015-10-01" ]
- }
- },
- "ioConfig" : {
- "type" : "hadoop",
- "inputSpec" : {
- "type" : "static",
- "inputFormat" : "org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat",
- "paths" : "example/book.seq"
- }
- },
- "tuningConfig" : {
- "type" : "hadoop",
- "partitionsSpec" : {
- "type" : "hashed",
- "targetPartitionSize" : 5000000
- },
- "jobProperties" : {
- "tmpjars": "/user/xxx/druid/test/book.jar"
- }
- }
- },
- "dataSource" : "test"
-}
diff --git a/extensions-contrib/thrift-extensions/pom.xml b/extensions-contrib/thrift-extensions/pom.xml
index 9e39c0b5d8b..30989f94374 100644
--- a/extensions-contrib/thrift-extensions/pom.xml
+++ b/extensions-contrib/thrift-extensions/pom.xml
@@ -65,12 +65,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-indexing-hadoop</artifactId>
- <version>${project.parent.version}</version>
- <scope>provided</scope>
- </dependency>
<dependency>
<groupId>com.twitter.elephantbird</groupId>
<artifactId>elephant-bird-core</artifactId>
diff --git a/extensions-core/avro-extensions/pom.xml b/extensions-core/avro-extensions/pom.xml
index a1eddb54f62..8aa5d559286 100644
--- a/extensions-core/avro-extensions/pom.xml
+++ b/extensions-core/avro-extensions/pom.xml
@@ -67,41 +67,6 @@
</exclusion>
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro-mapred</artifactId>
- <version>${avro.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-server</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.eclipse.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-lang3</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
<dependency>
<groupId>org.apache.druid</groupId>
<artifactId>druid-processing</artifactId>
@@ -278,12 +243,6 @@
<activeByDefault>true</activeByDefault>
</activation>
<dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client-api</artifactId>
- <version>${hadoop.compile.version}</version>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
diff --git a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/AvroHadoopInputRowParser.java b/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/AvroHadoopInputRowParser.java
deleted file mode 100644
index 39bdc87cc5b..00000000000
--- a/extensions-core/avro-extensions/src/main/java/org/apache/druid/data/input/AvroHadoopInputRowParser.java
+++ /dev/null
... 52424 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org