You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by vo...@apache.org on 2022/09/17 05:10:44 UTC
[druid-website-src] 01/01: doc build
This is an automated email from the ASF dual-hosted git repository.
vogievetsky pushed a commit to branch doc_polish_24
in repository https://gitbox.apache.org/repos/asf/druid-website-src.git
commit 8c78a7f61dfc45c0397d10e4831d03d6c3417ba9
Author: Vadim Ogievetsky <va...@ogievetsky.com>
AuthorDate: Fri Sep 16 22:10:11 2022 -0700
doc build
---
.../24.0.0/comparisons/druid-vs-elasticsearch.html | 2 +-
docs/24.0.0/comparisons/druid-vs-key-value.html | 2 +-
docs/24.0.0/comparisons/druid-vs-kudu.html | 2 +-
docs/24.0.0/comparisons/druid-vs-redshift.html | 2 +-
docs/24.0.0/comparisons/druid-vs-spark.html | 2 +-
.../24.0.0/comparisons/druid-vs-sql-on-hadoop.html | 2 +-
docs/24.0.0/configuration/index.html | 16 +-
docs/24.0.0/configuration/logging.html | 4 +-
.../data-management/automatic-compaction.html | 232 ++++++++++++++++
.../{ingestion => data-management}/compaction.html | 20 +-
docs/24.0.0/data-management/delete.html | 156 +++++++++++
docs/24.0.0/data-management/index.html | 109 ++++++++
docs/24.0.0/data-management/schema-changes.html | 110 ++++++++
docs/24.0.0/data-management/update.html | 138 ++++++++++
docs/24.0.0/dependencies/deep-storage.html | 2 +-
docs/24.0.0/dependencies/metadata-storage.html | 2 +-
docs/24.0.0/dependencies/zookeeper.html | 2 +-
docs/24.0.0/design/architecture.html | 8 +-
docs/24.0.0/design/auth.html | 2 +-
docs/24.0.0/design/broker.html | 2 +-
docs/24.0.0/design/coordinator.html | 6 +-
docs/24.0.0/design/historical.html | 2 +-
docs/24.0.0/design/index.html | 4 +-
docs/24.0.0/design/indexer.html | 3 +-
docs/24.0.0/design/indexing-service.html | 2 +-
docs/24.0.0/design/middlemanager.html | 2 +-
docs/24.0.0/design/overlord.html | 2 +-
docs/24.0.0/design/peons.html | 2 +-
docs/24.0.0/design/processes.html | 4 +-
docs/24.0.0/design/router.html | 4 +-
docs/24.0.0/design/segments.html | 4 +-
docs/24.0.0/development/build.html | 2 +-
docs/24.0.0/development/experimental.html | 2 +-
.../development/extensions-contrib/aliyun-oss.html | 2 +-
.../extensions-contrib/ambari-metrics-emitter.html | 2 +-
.../development/extensions-contrib/cassandra.html | 2 +-
.../development/extensions-contrib/cloudfiles.html | 2 +-
.../extensions-contrib/distinctcount.html | 2 +-
.../extensions-contrib/gce-extensions.html | 2 +-
.../development/extensions-contrib/graphite.html | 2 +-
.../development/extensions-contrib/influx.html | 2 +-
.../extensions-contrib/influxdb-emitter.html | 2 +-
.../extensions-contrib/kafka-emitter.html | 4 +-
.../extensions-contrib/materialized-view.html | 2 +-
.../extensions-contrib/momentsketch-quantiles.html | 2 +-
.../extensions-contrib/moving-average-query.html | 2 +-
.../extensions-contrib/opentsdb-emitter.html | 2 +-
.../development/extensions-contrib/prometheus.html | 2 +-
.../extensions-contrib/redis-cache.html | 2 +-
.../development/extensions-contrib/sqlserver.html | 2 +-
.../development/extensions-contrib/statsd.html | 2 +-
.../tdigestsketch-quantiles.html | 2 +-
.../development/extensions-contrib/thrift.html | 2 +-
.../extensions-contrib/time-min-max.html | 2 +-
.../extensions-core/approximate-histograms.html | 2 +-
docs/24.0.0/development/extensions-core/avro.html | 2 +-
docs/24.0.0/development/extensions-core/azure.html | 2 +-
.../development/extensions-core/bloom-filter.html | 2 +-
.../extensions-core/datasketches-extension.html | 2 +-
.../extensions-core/datasketches-hll.html | 2 +-
.../extensions-core/datasketches-quantiles.html | 2 +-
.../extensions-core/datasketches-theta.html | 2 +-
.../extensions-core/datasketches-tuple.html | 2 +-
.../development/extensions-core/druid-aws-rds.html | 2 +-
.../extensions-core/druid-basic-security.html | 2 +-
.../extensions-core/druid-kerberos.html | 2 +-
.../development/extensions-core/druid-lookups.html | 2 +-
.../development/extensions-core/druid-pac4j.html | 4 +-
.../extensions-core/druid-ranger-security.html | 2 +-
.../24.0.0/development/extensions-core/google.html | 2 +-
docs/24.0.0/development/extensions-core/hdfs.html | 2 +-
.../kafka-extraction-namespace.html | 2 +-
.../extensions-core/kafka-ingestion.html | 6 +-
.../kafka-supervisor-operations.html | 2 +-
.../kafka-supervisor-reference.html | 4 +-
.../extensions-core/kinesis-ingestion.html | 2 +-
.../development/extensions-core/kubernetes.html | 2 +-
.../extensions-core/lookups-cached-global.html | 2 +-
docs/24.0.0/development/extensions-core/mysql.html | 2 +-
docs/24.0.0/development/extensions-core/orc.html | 2 +-
.../development/extensions-core/parquet.html | 2 +-
.../development/extensions-core/postgresql.html | 2 +-
.../development/extensions-core/protobuf.html | 2 +-
docs/24.0.0/development/extensions-core/s3.html | 2 +-
.../extensions-core/simple-client-sslcontext.html | 2 +-
docs/24.0.0/development/extensions-core/stats.html | 2 +-
.../development/extensions-core/test-stats.html | 2 +-
docs/24.0.0/development/extensions.html | 2 +-
docs/24.0.0/development/geo.html | 2 +-
docs/24.0.0/development/javascript.html | 2 +-
docs/24.0.0/development/modules.html | 4 +-
docs/24.0.0/development/overview.html | 2 +-
docs/24.0.0/development/versioning.html | 2 +-
docs/24.0.0/ingestion/automatic-compaction.html | 240 +----------------
docs/24.0.0/ingestion/compaction.html | 297 +--------------------
docs/24.0.0/ingestion/data-formats.html | 6 +-
docs/24.0.0/ingestion/data-management.html | 176 +-----------
docs/24.0.0/ingestion/data-model.html | 4 +-
docs/24.0.0/ingestion/delete-data.html | 8 +-
docs/24.0.0/ingestion/faq.html | 22 +-
docs/24.0.0/ingestion/hadoop.html | 4 +-
docs/24.0.0/ingestion/index.html | 52 ++--
docs/24.0.0/ingestion/ingestion-spec.html | 16 +-
docs/24.0.0/ingestion/native-batch-firehose.html | 4 +-
.../ingestion/native-batch-input-sources.html | 16 +-
.../24.0.0/ingestion/native-batch-simple-task.html | 21 +-
docs/24.0.0/ingestion/native-batch.html | 33 ++-
docs/24.0.0/ingestion/partitioning.html | 22 +-
docs/24.0.0/ingestion/rollup.html | 5 +-
docs/24.0.0/ingestion/schema-changes.html | 8 +
docs/24.0.0/ingestion/schema-design.html | 4 +-
docs/24.0.0/ingestion/standalone-realtime.html | 4 +-
docs/24.0.0/ingestion/tasks.html | 25 +-
docs/24.0.0/ingestion/update-existing-data.html | 8 +-
docs/24.0.0/misc/math-expr.html | 2 +-
docs/24.0.0/misc/papers-and-talks.html | 2 +-
docs/24.0.0/multi-stage-query/api.html | 42 +--
docs/24.0.0/multi-stage-query/concepts.html | 274 +++++++++++--------
.../multi-stage-query/connect-external-data.html | 201 --------------
.../multi-stage-query/convert-json-spec.html | 233 ----------------
docs/24.0.0/multi-stage-query/examples.html | 19 +-
docs/24.0.0/multi-stage-query/index.html | 285 +++-----------------
docs/24.0.0/multi-stage-query/known-issues.html | 31 ++-
docs/24.0.0/multi-stage-query/reference.html | 228 +++++++++++-----
docs/24.0.0/multi-stage-query/security.html | 36 +--
docs/24.0.0/operations/alerts.html | 2 +-
docs/24.0.0/operations/api-reference.html | 7 +-
docs/24.0.0/operations/auth-ldap.html | 4 +-
docs/24.0.0/operations/basic-cluster-tuning.html | 4 +-
docs/24.0.0/operations/clean-metadata-store.html | 2 +-
docs/24.0.0/operations/deep-storage-migration.html | 2 +-
docs/24.0.0/operations/druid-console.html | 222 +--------------
docs/24.0.0/operations/dump-segment.html | 2 +-
.../24.0.0/operations/dynamic-config-provider.html | 2 +-
docs/24.0.0/operations/export-metadata.html | 2 +-
docs/24.0.0/operations/getting-started.html | 60 +----
docs/24.0.0/operations/high-availability.html | 2 +-
docs/24.0.0/operations/http-compression.html | 2 +-
docs/24.0.0/operations/insert-segment-to-db.html | 2 +-
docs/24.0.0/operations/java.html | 4 +-
docs/24.0.0/operations/kubernetes.html | 2 +-
docs/24.0.0/operations/management-uis.html | 8 +
docs/24.0.0/operations/metadata-migration.html | 2 +-
docs/24.0.0/operations/metrics.html | 2 +-
docs/24.0.0/operations/mixed-workloads.html | 2 +-
docs/24.0.0/operations/other-hadoop.html | 2 +-
docs/24.0.0/operations/password-provider.html | 2 +-
docs/24.0.0/operations/pull-deps.html | 2 +-
docs/24.0.0/operations/request-logging.html | 2 +-
docs/24.0.0/operations/reset-cluster.html | 2 +-
docs/24.0.0/operations/rolling-updates.html | 2 +-
docs/24.0.0/operations/rule-configuration.html | 4 +-
docs/24.0.0/operations/security-overview.html | 6 +-
docs/24.0.0/operations/security-user-auth.html | 8 +-
docs/24.0.0/operations/segment-optimization.html | 12 +-
docs/24.0.0/operations/single-server.html | 4 +-
docs/24.0.0/operations/tls-support.html | 2 +-
.../operations/use_sbt_to_build_fat_jar.html | 2 +-
docs/24.0.0/operations/web-console.html | 215 +++++++++++++++
docs/24.0.0/querying/aggregations.html | 2 +-
docs/24.0.0/querying/caching.html | 2 +-
docs/24.0.0/querying/datasource.html | 2 +-
docs/24.0.0/querying/datasourcemetadataquery.html | 2 +-
docs/24.0.0/querying/dimensionspecs.html | 5 +-
docs/24.0.0/querying/filters.html | 2 +-
docs/24.0.0/querying/granularities.html | 2 +-
docs/24.0.0/querying/groupbyquery.html | 2 +-
docs/24.0.0/querying/having.html | 2 +-
docs/24.0.0/querying/hll-old.html | 2 +-
docs/24.0.0/querying/joins.html | 2 +-
docs/24.0.0/querying/limitspec.html | 2 +-
docs/24.0.0/querying/lookups.html | 8 +-
docs/24.0.0/querying/multi-value-dimensions.html | 2 +-
docs/24.0.0/querying/multitenancy.html | 2 +-
docs/24.0.0/querying/nested-columns.html | 2 +-
docs/24.0.0/querying/post-aggregations.html | 2 +-
docs/24.0.0/querying/query-context.html | 2 +-
docs/24.0.0/querying/query-execution.html | 2 +-
docs/24.0.0/querying/querying.html | 4 +-
docs/24.0.0/querying/scan-query.html | 2 +-
docs/24.0.0/querying/searchquery.html | 2 +-
docs/24.0.0/querying/segmentmetadataquery.html | 2 +-
docs/24.0.0/querying/select-query.html | 4 +-
docs/24.0.0/querying/sorting-orders.html | 2 +-
docs/24.0.0/querying/sql-aggregations.html | 6 +-
docs/24.0.0/querying/sql-api.html | 4 +-
docs/24.0.0/querying/sql-data-types.html | 2 +-
docs/24.0.0/querying/sql-functions.html | 4 +-
docs/24.0.0/querying/sql-jdbc.html | 2 +-
docs/24.0.0/querying/sql-json-functions.html | 12 +-
docs/24.0.0/querying/sql-metadata-tables.html | 2 +-
.../querying/sql-multivalue-string-functions.html | 4 +-
docs/24.0.0/querying/sql-operators.html | 2 +-
docs/24.0.0/querying/sql-query-context.html | 2 +-
docs/24.0.0/querying/sql-scalar.html | 2 +-
docs/24.0.0/querying/sql-translation.html | 2 +-
docs/24.0.0/querying/sql.html | 4 +-
docs/24.0.0/querying/timeboundaryquery.html | 2 +-
docs/24.0.0/querying/timeseriesquery.html | 2 +-
docs/24.0.0/querying/topnmetricspec.html | 2 +-
docs/24.0.0/querying/topnquery.html | 2 +-
docs/24.0.0/querying/troubleshooting.html | 2 +-
docs/24.0.0/querying/using-caching.html | 2 +-
docs/24.0.0/querying/virtual-columns.html | 2 +-
docs/24.0.0/tutorials/cluster.html | 4 +-
docs/24.0.0/tutorials/docker.html | 12 +-
docs/24.0.0/tutorials/index.html | 65 +++--
docs/24.0.0/tutorials/tutorial-batch-hadoop.html | 2 +-
docs/24.0.0/tutorials/tutorial-batch-native.html | 4 +-
docs/24.0.0/tutorials/tutorial-batch.html | 13 +-
docs/24.0.0/tutorials/tutorial-compaction.html | 8 +-
docs/24.0.0/tutorials/tutorial-delete-data.html | 2 +-
docs/24.0.0/tutorials/tutorial-ingestion-spec.html | 2 +-
docs/24.0.0/tutorials/tutorial-kafka.html | 6 +-
.../24.0.0/tutorials/tutorial-kerberos-hadoop.html | 4 +-
.../tutorials/tutorial-msq-convert-json.html | 117 --------
.../tutorials/tutorial-msq-convert-spec.html | 235 ++++++++++++++++
docs/24.0.0/tutorials/tutorial-msq-extern.html | 203 ++++++++++++++
.../tutorials/tutorial-msq-external-data.html | 117 --------
docs/24.0.0/tutorials/tutorial-query.html | 14 +-
docs/24.0.0/tutorials/tutorial-retention.html | 6 +-
docs/24.0.0/tutorials/tutorial-rollup.html | 2 +-
docs/24.0.0/tutorials/tutorial-sketches-theta.html | 4 +-
docs/24.0.0/tutorials/tutorial-transform-spec.html | 4 +-
docs/24.0.0/tutorials/tutorial-update-data.html | 2 +-
.../latest/comparisons/druid-vs-elasticsearch.html | 2 +-
docs/latest/comparisons/druid-vs-key-value.html | 2 +-
docs/latest/comparisons/druid-vs-kudu.html | 2 +-
docs/latest/comparisons/druid-vs-redshift.html | 2 +-
docs/latest/comparisons/druid-vs-spark.html | 2 +-
.../latest/comparisons/druid-vs-sql-on-hadoop.html | 2 +-
docs/latest/configuration/index.html | 16 +-
docs/latest/configuration/logging.html | 4 +-
.../data-management/automatic-compaction.html | 232 ++++++++++++++++
.../{ingestion => data-management}/compaction.html | 20 +-
docs/latest/data-management/delete.html | 156 +++++++++++
docs/latest/data-management/index.html | 109 ++++++++
docs/latest/data-management/schema-changes.html | 110 ++++++++
docs/latest/data-management/update.html | 138 ++++++++++
docs/latest/dependencies/deep-storage.html | 2 +-
docs/latest/dependencies/metadata-storage.html | 2 +-
docs/latest/dependencies/zookeeper.html | 2 +-
docs/latest/design/architecture.html | 8 +-
docs/latest/design/auth.html | 2 +-
docs/latest/design/broker.html | 2 +-
docs/latest/design/coordinator.html | 6 +-
docs/latest/design/historical.html | 2 +-
docs/latest/design/index.html | 4 +-
docs/latest/design/indexer.html | 3 +-
docs/latest/design/indexing-service.html | 2 +-
docs/latest/design/middlemanager.html | 2 +-
docs/latest/design/overlord.html | 2 +-
docs/latest/design/peons.html | 2 +-
docs/latest/design/processes.html | 4 +-
docs/latest/design/router.html | 4 +-
docs/latest/design/segments.html | 4 +-
docs/latest/development/build.html | 2 +-
docs/latest/development/experimental.html | 2 +-
.../development/extensions-contrib/aliyun-oss.html | 2 +-
.../extensions-contrib/ambari-metrics-emitter.html | 2 +-
.../development/extensions-contrib/cassandra.html | 2 +-
.../development/extensions-contrib/cloudfiles.html | 2 +-
.../extensions-contrib/distinctcount.html | 2 +-
.../extensions-contrib/gce-extensions.html | 2 +-
.../development/extensions-contrib/graphite.html | 2 +-
.../development/extensions-contrib/influx.html | 2 +-
.../extensions-contrib/influxdb-emitter.html | 2 +-
.../extensions-contrib/kafka-emitter.html | 4 +-
.../extensions-contrib/materialized-view.html | 2 +-
.../extensions-contrib/momentsketch-quantiles.html | 2 +-
.../extensions-contrib/moving-average-query.html | 2 +-
.../extensions-contrib/opentsdb-emitter.html | 2 +-
.../development/extensions-contrib/prometheus.html | 2 +-
.../extensions-contrib/redis-cache.html | 2 +-
.../development/extensions-contrib/sqlserver.html | 2 +-
.../development/extensions-contrib/statsd.html | 2 +-
.../tdigestsketch-quantiles.html | 2 +-
.../development/extensions-contrib/thrift.html | 2 +-
.../extensions-contrib/time-min-max.html | 2 +-
.../extensions-core/approximate-histograms.html | 2 +-
docs/latest/development/extensions-core/avro.html | 2 +-
docs/latest/development/extensions-core/azure.html | 2 +-
.../development/extensions-core/bloom-filter.html | 2 +-
.../extensions-core/datasketches-extension.html | 2 +-
.../extensions-core/datasketches-hll.html | 2 +-
.../extensions-core/datasketches-quantiles.html | 2 +-
.../extensions-core/datasketches-theta.html | 2 +-
.../extensions-core/datasketches-tuple.html | 2 +-
.../development/extensions-core/druid-aws-rds.html | 2 +-
.../extensions-core/druid-basic-security.html | 2 +-
.../extensions-core/druid-kerberos.html | 2 +-
.../development/extensions-core/druid-lookups.html | 2 +-
.../development/extensions-core/druid-pac4j.html | 4 +-
.../extensions-core/druid-ranger-security.html | 2 +-
.../latest/development/extensions-core/google.html | 2 +-
docs/latest/development/extensions-core/hdfs.html | 2 +-
.../kafka-extraction-namespace.html | 2 +-
.../extensions-core/kafka-ingestion.html | 6 +-
.../kafka-supervisor-operations.html | 2 +-
.../kafka-supervisor-reference.html | 4 +-
.../extensions-core/kinesis-ingestion.html | 2 +-
.../development/extensions-core/kubernetes.html | 2 +-
.../extensions-core/lookups-cached-global.html | 2 +-
docs/latest/development/extensions-core/mysql.html | 2 +-
docs/latest/development/extensions-core/orc.html | 2 +-
.../development/extensions-core/parquet.html | 2 +-
.../development/extensions-core/postgresql.html | 2 +-
.../development/extensions-core/protobuf.html | 2 +-
docs/latest/development/extensions-core/s3.html | 2 +-
.../extensions-core/simple-client-sslcontext.html | 2 +-
docs/latest/development/extensions-core/stats.html | 2 +-
.../development/extensions-core/test-stats.html | 2 +-
docs/latest/development/extensions.html | 2 +-
docs/latest/development/geo.html | 2 +-
docs/latest/development/javascript.html | 2 +-
docs/latest/development/modules.html | 4 +-
docs/latest/development/overview.html | 2 +-
docs/latest/development/versioning.html | 2 +-
docs/latest/ingestion/automatic-compaction.html | 240 +----------------
docs/latest/ingestion/compaction.html | 297 +--------------------
docs/latest/ingestion/data-formats.html | 6 +-
docs/latest/ingestion/data-management.html | 176 +-----------
docs/latest/ingestion/data-model.html | 4 +-
docs/latest/ingestion/delete-data.html | 8 +-
docs/latest/ingestion/faq.html | 22 +-
docs/latest/ingestion/hadoop.html | 4 +-
docs/latest/ingestion/index.html | 52 ++--
docs/latest/ingestion/ingestion-spec.html | 16 +-
docs/latest/ingestion/native-batch-firehose.html | 4 +-
.../ingestion/native-batch-input-sources.html | 16 +-
.../latest/ingestion/native-batch-simple-task.html | 21 +-
docs/latest/ingestion/native-batch.html | 33 ++-
docs/latest/ingestion/partitioning.html | 22 +-
docs/latest/ingestion/rollup.html | 5 +-
docs/latest/ingestion/schema-changes.html | 8 +
docs/latest/ingestion/schema-design.html | 4 +-
docs/latest/ingestion/standalone-realtime.html | 4 +-
docs/latest/ingestion/tasks.html | 25 +-
docs/latest/ingestion/update-existing-data.html | 8 +-
docs/latest/misc/math-expr.html | 2 +-
docs/latest/misc/papers-and-talks.html | 2 +-
docs/latest/multi-stage-query/api.html | 42 +--
docs/latest/multi-stage-query/concepts.html | 274 +++++++++++--------
.../multi-stage-query/connect-external-data.html | 201 --------------
.../multi-stage-query/convert-json-spec.html | 233 ----------------
docs/latest/multi-stage-query/examples.html | 19 +-
docs/latest/multi-stage-query/index.html | 285 +++-----------------
docs/latest/multi-stage-query/known-issues.html | 31 ++-
docs/latest/multi-stage-query/reference.html | 228 +++++++++++-----
docs/latest/multi-stage-query/security.html | 36 +--
docs/latest/operations/alerts.html | 2 +-
docs/latest/operations/api-reference.html | 7 +-
docs/latest/operations/auth-ldap.html | 4 +-
docs/latest/operations/basic-cluster-tuning.html | 4 +-
docs/latest/operations/clean-metadata-store.html | 2 +-
docs/latest/operations/deep-storage-migration.html | 2 +-
docs/latest/operations/druid-console.html | 222 +--------------
docs/latest/operations/dump-segment.html | 2 +-
.../latest/operations/dynamic-config-provider.html | 2 +-
docs/latest/operations/export-metadata.html | 2 +-
docs/latest/operations/getting-started.html | 60 +----
docs/latest/operations/high-availability.html | 2 +-
docs/latest/operations/http-compression.html | 2 +-
docs/latest/operations/insert-segment-to-db.html | 2 +-
docs/latest/operations/java.html | 4 +-
docs/latest/operations/kubernetes.html | 2 +-
docs/latest/operations/management-uis.html | 8 +
docs/latest/operations/metadata-migration.html | 2 +-
docs/latest/operations/metrics.html | 2 +-
docs/latest/operations/mixed-workloads.html | 2 +-
docs/latest/operations/other-hadoop.html | 2 +-
docs/latest/operations/password-provider.html | 2 +-
docs/latest/operations/pull-deps.html | 2 +-
docs/latest/operations/request-logging.html | 2 +-
docs/latest/operations/reset-cluster.html | 2 +-
docs/latest/operations/rolling-updates.html | 2 +-
docs/latest/operations/rule-configuration.html | 4 +-
docs/latest/operations/security-overview.html | 6 +-
docs/latest/operations/security-user-auth.html | 8 +-
docs/latest/operations/segment-optimization.html | 12 +-
docs/latest/operations/single-server.html | 4 +-
docs/latest/operations/tls-support.html | 2 +-
.../operations/use_sbt_to_build_fat_jar.html | 2 +-
docs/latest/operations/web-console.html | 215 +++++++++++++++
docs/latest/querying/aggregations.html | 2 +-
docs/latest/querying/caching.html | 2 +-
docs/latest/querying/datasource.html | 2 +-
docs/latest/querying/datasourcemetadataquery.html | 2 +-
docs/latest/querying/dimensionspecs.html | 5 +-
docs/latest/querying/filters.html | 2 +-
docs/latest/querying/granularities.html | 2 +-
docs/latest/querying/groupbyquery.html | 2 +-
docs/latest/querying/having.html | 2 +-
docs/latest/querying/hll-old.html | 2 +-
docs/latest/querying/joins.html | 2 +-
docs/latest/querying/limitspec.html | 2 +-
docs/latest/querying/lookups.html | 8 +-
docs/latest/querying/multi-value-dimensions.html | 2 +-
docs/latest/querying/multitenancy.html | 2 +-
docs/latest/querying/nested-columns.html | 2 +-
docs/latest/querying/post-aggregations.html | 2 +-
docs/latest/querying/query-context.html | 2 +-
docs/latest/querying/query-execution.html | 2 +-
docs/latest/querying/querying.html | 4 +-
docs/latest/querying/scan-query.html | 2 +-
docs/latest/querying/searchquery.html | 2 +-
docs/latest/querying/segmentmetadataquery.html | 2 +-
docs/latest/querying/select-query.html | 4 +-
docs/latest/querying/sorting-orders.html | 2 +-
docs/latest/querying/sql-aggregations.html | 6 +-
docs/latest/querying/sql-api.html | 4 +-
docs/latest/querying/sql-data-types.html | 2 +-
docs/latest/querying/sql-functions.html | 4 +-
docs/latest/querying/sql-jdbc.html | 2 +-
docs/latest/querying/sql-json-functions.html | 12 +-
docs/latest/querying/sql-metadata-tables.html | 2 +-
.../querying/sql-multivalue-string-functions.html | 4 +-
docs/latest/querying/sql-operators.html | 2 +-
docs/latest/querying/sql-query-context.html | 2 +-
docs/latest/querying/sql-scalar.html | 2 +-
docs/latest/querying/sql-translation.html | 2 +-
docs/latest/querying/sql.html | 4 +-
docs/latest/querying/timeboundaryquery.html | 2 +-
docs/latest/querying/timeseriesquery.html | 2 +-
docs/latest/querying/topnmetricspec.html | 2 +-
docs/latest/querying/topnquery.html | 2 +-
docs/latest/querying/troubleshooting.html | 2 +-
docs/latest/querying/using-caching.html | 2 +-
docs/latest/querying/virtual-columns.html | 2 +-
docs/latest/tutorials/cluster.html | 4 +-
docs/latest/tutorials/docker.html | 12 +-
docs/latest/tutorials/index.html | 65 +++--
docs/latest/tutorials/tutorial-batch-hadoop.html | 2 +-
docs/latest/tutorials/tutorial-batch-native.html | 4 +-
docs/latest/tutorials/tutorial-batch.html | 13 +-
docs/latest/tutorials/tutorial-compaction.html | 8 +-
docs/latest/tutorials/tutorial-delete-data.html | 2 +-
docs/latest/tutorials/tutorial-ingestion-spec.html | 2 +-
docs/latest/tutorials/tutorial-kafka.html | 6 +-
.../latest/tutorials/tutorial-kerberos-hadoop.html | 4 +-
.../tutorials/tutorial-msq-convert-json.html | 117 --------
.../tutorials/tutorial-msq-convert-spec.html | 235 ++++++++++++++++
docs/latest/tutorials/tutorial-msq-extern.html | 203 ++++++++++++++
.../tutorials/tutorial-msq-external-data.html | 117 --------
docs/latest/tutorials/tutorial-query.html | 14 +-
docs/latest/tutorials/tutorial-retention.html | 6 +-
docs/latest/tutorials/tutorial-rollup.html | 2 +-
docs/latest/tutorials/tutorial-sketches-theta.html | 4 +-
docs/latest/tutorials/tutorial-transform-spec.html | 4 +-
docs/latest/tutorials/tutorial-update-data.html | 2 +-
450 files changed, 4608 insertions(+), 5122 deletions(-)
diff --git a/docs/24.0.0/comparisons/druid-vs-elasticsearch.html b/docs/24.0.0/comparisons/druid-vs-elasticsearch.html
index dfe91359..8f42c906 100644
--- a/docs/24.0.0/comparisons/druid-vs-elasticsearch.html
+++ b/docs/24.0.0/comparisons/druid-vs-elasticsearch.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/comparisons/druid-vs-key-value.html b/docs/24.0.0/comparisons/druid-vs-key-value.html
index a31cb07c..1ca0ef3a 100644
--- a/docs/24.0.0/comparisons/druid-vs-key-value.html
+++ b/docs/24.0.0/comparisons/druid-vs-key-value.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/comparisons/druid-vs-kudu.html b/docs/24.0.0/comparisons/druid-vs-kudu.html
index fb636321..da09a295 100644
--- a/docs/24.0.0/comparisons/druid-vs-kudu.html
+++ b/docs/24.0.0/comparisons/druid-vs-kudu.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/comparisons/druid-vs-redshift.html b/docs/24.0.0/comparisons/druid-vs-redshift.html
index d7c8e1e8..92bb8870 100644
--- a/docs/24.0.0/comparisons/druid-vs-redshift.html
+++ b/docs/24.0.0/comparisons/druid-vs-redshift.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/comparisons/druid-vs-spark.html b/docs/24.0.0/comparisons/druid-vs-spark.html
index 09160944..c190d974 100644
--- a/docs/24.0.0/comparisons/druid-vs-spark.html
+++ b/docs/24.0.0/comparisons/druid-vs-spark.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/comparisons/druid-vs-sql-on-hadoop.html b/docs/24.0.0/comparisons/druid-vs-sql-on-hadoop.html
index 0e9f9d7a..5017605a 100644
--- a/docs/24.0.0/comparisons/druid-vs-sql-on-hadoop.html
+++ b/docs/24.0.0/comparisons/druid-vs-sql-on-hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/configuration/index.html b/docs/24.0.0/configuration/index.html
index 046cb7ce..876353b8 100644
--- a/docs/24.0.0/configuration/index.html
+++ b/docs/24.0.0/configuration/index.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -1027,7 +1027,7 @@ Support for 64-bit floating point columns was released in Druid 0.11.0, so if yo
</table>
<h4><a class="anchor" aria-hidden="true" id="dynamic-configuration"></a><a href="#dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
<p>The Coordinator has dynamic configuration to change certain behavior on the fly.</p>
-<p>It is recommended that you use the <a href="/docs/24.0.0/operations/druid-console.html">web console</a> to configure these parameters.
+<p>It is recommended that you use the <a href="/docs/24.0.0/operations/web-console.html">web console</a> to configure these parameters.
However, if you need to do it via HTTP, the JSON object can be submitted to the Coordinator via a POST request at:</p>
<pre><code class="hljs">http:<span class="hljs-regexp">//</span><COORDINATOR_IP>:<PORT><span class="hljs-regexp">/druid/</span>coordinator<span class="hljs-regexp">/v1/</span>config
</code></pre>
@@ -1109,7 +1109,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
</tbody>
</table>
<h5><a class="anchor" aria-hidden="true" id="automatic-compaction-dynamic-configuration"></a><a href="#automatic-compaction-dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c [...]
-<p>You can set or update <a href="/docs/24.0.0/ingestion/automatic-compaction.html">automatic compaction</a> properties dynamically using the
+<p>You can set or update <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a> properties dynamically using the
<a href="/docs/24.0.0/operations/api-reference.html#automatic-compaction-configuration">Coordinator API</a> without restarting Coordinators.</p>
<p>For details about segment compaction, see <a href="/docs/24.0.0/operations/segment-optimization.html">Segment size optimization</a>.</p>
<p>You can configure automatic compaction through the following properties:</p>
@@ -1121,7 +1121,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
<tr><td><code>dataSource</code></td><td>dataSource name to be compacted.</td><td>yes</td></tr>
<tr><td><code>taskPriority</code></td><td><a href="/docs/24.0.0/ingestion/tasks.html#priority">Priority</a> of compaction task.</td><td>no (default = 25)</td></tr>
<tr><td><code>inputSegmentSizeBytes</code></td><td>Maximum number of total segment bytes processed per compaction task. Since a time chunk must be processed in its entirety, if the segments for a particular time chunk have a total size in bytes greater than this parameter, compaction will not run for that time chunk. Because each compaction task runs with a single thread, setting this value too far above 1–2GB will result in compaction tasks taking an excessive amount of time.</td><td>no [...]
-<tr><td><code>skipOffsetFromLatest</code></td><td>The offset for searching segments to be compacted in <a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a> duration format. Strongly recommended to set for realtime dataSources. See <a href="/docs/24.0.0/ingestion/compaction.html#data-handling-with-compaction">Data handling with compaction</a>.</td><td>no (default = "P1D")</td></tr>
+<tr><td><code>skipOffsetFromLatest</code></td><td>The offset for searching segments to be compacted in <a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a> duration format. Strongly recommended to set for realtime dataSources. See <a href="/docs/24.0.0/data-management/compaction.html#data-handling-with-compaction">Data handling with compaction</a>.</td><td>no (default = "P1D")</td></tr>
<tr><td><code>tuningConfig</code></td><td>Tuning config for compaction tasks. See below <a href="#automatic-compaction-tuningconfig">Automatic compaction tuningConfig</a>.</td><td>no</td></tr>
<tr><td><code>taskContext</code></td><td><a href="/docs/24.0.0/ingestion/tasks.html#context">Task context</a> for compaction tasks.</td><td>no</td></tr>
<tr><td><code>granularitySpec</code></td><td>Custom <code>granularitySpec</code>. See <a href="#automatic-compaction-granularityspec">Automatic compaction granularitySpec</a>.</td><td>No</td></tr>
@@ -1143,7 +1143,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
You may see this issue with streaming ingestion from Kafka and Kinesis, which ingest late-arriving data.</p>
<p>To mitigate this problem, set <code>skipOffsetFromLatest</code> to a value large enough so that arriving data tends to fall outside the offset value from the current time. This way you can avoid conflicts between compaction tasks and realtime ingestion tasks.
For example, if you want to skip over segments from thirty days prior to the end time of the most recent segment, assign <code>"skipOffsetFromLatest": "P30D"</code>.
-For more information, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
+For more information, see <a href="/docs/24.0.0/data-management/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
<h6><a class="anchor" aria-hidden="true" id="automatic-compaction-tuningconfig"></a><a href="#automatic-compaction-tuningconfig" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S1 [...]
<p>Auto-compaction supports a subset of the <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig">tuningConfig for Parallel task</a>.
The below is a list of the supported configurations for auto-compaction.</p>
@@ -1161,7 +1161,7 @@ The below is a list of the supported configurations for auto-compaction.</p>
<tr><td><code>indexSpecForIntermediatePersists</code></td><td>Defines segment storage format options to be used at indexing time for intermediate persisted temporary segments. this can be used to disable dimension/metric compression on intermediate segments to reduce memory required for final merging. however, disabling compression on intermediate segments might increase page cache use while they are used before getting merged into final segment published, see <a href="/docs/24.0.0/inges [...]
<tr><td><code>maxPendingPersists</code></td><td>Maximum number of persists that can be pending but not started. If this limit would be exceeded by a new intermediate persist, ingestion will block until the currently-running persist finishes. Maximum heap memory usage for indexing scales with <code>maxRowsInMemory</code> * (2 + <code>maxPendingPersists</code>).</td><td>no (default = 0, meaning one persist can be running concurrently with ingestion, and none can be queued up)</td></tr>
<tr><td><code>pushTimeout</code></td><td>Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.</td><td>no (default = 0)</td></tr>
-<tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="/docs/24.0.0/ingestion/native-batch-simple-task.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>no (default is the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used)</td></tr>
+<tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="/docs/24.0.0/ingestion/native-batch.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>no (default is the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used)</td></tr>
<tr><td><code>maxNumConcurrentSubTasks</code></td><td>Maximum number of worker tasks which can be run in parallel at the same time. The supervisor task would spawn worker tasks up to <code>maxNumConcurrentSubTasks</code> regardless of the current available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion. [...]
<tr><td><code>maxRetry</code></td><td>Maximum number of retries on task failures.</td><td>no (default = 3)</td></tr>
<tr><td><code>maxNumSegmentsToMerge</code></td><td>Max limit for the number of segments that a single task can merge at the same time in the second phase. Used only with <code>hashed</code> or <code>single_dim</code> partitionsSpec.</td><td>no (default = 100)</td></tr>
@@ -1965,7 +1965,7 @@ line.</p>
<tr><td><code>druid.server.http.enableRequestLimit</code></td><td>If enabled, no requests would be queued in jetty queue and "HTTP 429 Too Many Requests" error response would be sent.</td><td>false</td></tr>
<tr><td><code>druid.server.http.defaultQueryTimeout</code></td><td>Query timeout in millis, beyond which unfinished queries will be cancelled</td><td>300000</td></tr>
<tr><td><code>druid.server.http.maxScatterGatherBytes</code></td><td>Maximum number of bytes gathered from data processes such as Historicals and realtime processes to execute a query. Queries that exceed this limit will fail. This is an advance configuration that allows to protect in case Broker is under heavy load and not utilizing the data gathered in memory fast enough and leading to OOMs. This limit can be further reduced at query time using <code>maxScatterGatherBytes</code> in the [...]
-<tr><td><code>druid.server.http.maxSubqueryRows</code></td><td>Maximum number of rows from all subqueries per query. Druid stores the subquery rows in temporary tables that live in the Java heap. <code>druid.server.http.maxSubqueryRows</code> is a guardrail to prevent the system from exhausting available heap. When a subquery exceeds the row limit, Druid throws a resource limit exceeded exception: "Subquery generated results beyond maximum."<br><br>It is a good practice to avoi [...]
+<tr><td><code>druid.server.http.maxSubqueryRows</code></td><td>Maximum number of rows from all subqueries per query. Druid stores the subquery rows in temporary tables that live in the Java heap. <code>druid.server.http.maxSubqueryRows</code> is a guardrail to prevent the system from exhausting available heap. When a subquery exceeds the row limit, Druid throws a resource limit exceeded exception: "Subquery generated results beyond maximum."<br /><br />It is a good practice to [...]
<tr><td><code>druid.server.http.gracefulShutdownTimeout</code></td><td>The maximum amount of time Jetty waits after receiving shutdown signal. After this timeout the threads will be forcefully shutdown. This allows any queries that are executing to complete(Only values greater than zero are valid).</td><td><code>PT30S</code></td></tr>
<tr><td><code>druid.server.http.unannouncePropagationDelay</code></td><td>How long to wait for ZooKeeper unannouncements to propagate before shutting down Jetty. This is a minimum and <code>druid.server.http.gracefulShutdownTimeout</code> does not start counting down until after this period elapses.</td><td><code>PT0S</code> (do not wait)</td></tr>
<tr><td><code>druid.server.http.maxQueryTimeout</code></td><td>Maximum allowed value (in milliseconds) for <code>timeout</code> parameter. See <a href="/docs/24.0.0/querying/query-context.html">query-context</a> to know more about <code>timeout</code>. Query is rejected if the query context <code>timeout</code> is greater than this value.</td><td>Long.MAX_VALUE</td></tr>
@@ -2070,7 +2070,7 @@ queries in order to avoid running as a default priority of 0.</p>
<tr><td><code>druid.sql.planner.authorizeSystemTablesDirectly</code></td><td>If true, Druid authorizes queries against any of the system schema tables (<code>sys</code> in SQL) as <code>SYSTEM_TABLE</code> resources which require <code>READ</code> access, in addition to permissions based content filtering.</td><td>false</td></tr>
<tr><td><code>druid.sql.planner.useNativeQueryExplain</code></td><td>If true, <code>EXPLAIN PLAN FOR</code> will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with <code>useNativeQueryExplain</code> context key.</td><td>true</td></tr>
<tr><td><code>druid.sql.planner.maxNumericInFilters</code></td><td>Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an <a href="/docs/24.0.0/querying/filters.html#or">OR</a> of <a href="/docs/24.0.0/querying/filters.html#bound-filter">Bound filter</a>. By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries [...]
-<tr><td><code>druid.sql.approxCountDistinct.function</code></td><td>Implementation to use for the <a href="/docs/24.0.0/querying/sql-aggregations.html"><code>APPROX_COUNT_DISTINCT</code> function</a>. Without extensions loaded, the only valid value is <code>APPROX_COUNT_DISTINCT_BUILTIN</code> (a HyperLogLog, or HLL, based implementation). If the <a href="/docs/24.0.0/development/extensions-core/datasketches-extension.html">DataSketches extension</a> is loaded, this can also be <code>APP [...]
+<tr><td><code>druid.sql.approxCountDistinct.function</code></td><td>Implementation to use for the <a href="/docs/24.0.0/querying/sql-aggregations.html"><code>APPROX_COUNT_DISTINCT</code> function</a>. Without extensions loaded, the only valid value is <code>APPROX_COUNT_DISTINCT_BUILTIN</code> (a HyperLogLog, or HLL, based implementation). If the <a href="/docs/24.0.0/development/extensions-core/datasketches-extension.html">DataSketches extension</a> is loaded, this can also be <code>APP [...]
</tbody>
</table>
<blockquote>
diff --git a/docs/24.0.0/configuration/logging.html b/docs/24.0.0/configuration/logging.html
index a4b81668..6537ea26 100644
--- a/docs/24.0.0/configuration/logging.html
+++ b/docs/24.0.0/configuration/logging.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -176,7 +176,7 @@ about <a href="/docs/24.0.0/operations/java.html#strong-encapsulation">Java stro
<span class="hljs-tag"></<span class="hljs-name">Loggers</span>></span>
<span class="hljs-tag"></<span class="hljs-name">Configuration</span>></span>
</span></code></pre>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/development/extensions.html"><span class="arrow-prev">← </span><span>Extensions</span></a><a class="docs-next button" href="/docs/24.0.0/operations/druid-console.html"><span>Druid console</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#log-directory">Log directory</a></li><li><a href="#all-in-one-start-commands" [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/development/extensions.html"><span class="arrow-prev">← </span><span>Extensions</span></a><a class="docs-next button" href="/docs/24.0.0/operations/web-console.html"><span>Web console</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#log-directory">Log directory</a></li><li><a href="#all-in-one-start-commands">All [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/data-management/automatic-compaction.html b/docs/24.0.0/data-management/automatic-compaction.html
new file mode 100644
index 00000000..cff48448
--- /dev/null
+++ b/docs/24.0.0/data-management/automatic-compaction.html
@@ -0,0 +1,232 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Automatic compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/automatic-compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" c [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/automatic-compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Automatic compaction</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<p>In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to <a href="/docs/24.0.0/operations/segment-optimization.html">optimally size segments</a> after ingestion to improve query performance. Automatic compaction, or auto-compaction, refers to the system for automatic execution of compaction tasks managed by the <a href="/docs/24.0.0/design/coordinator.html">Drui [...]
+<p>The Coordinator <a href="/docs/24.0.0/configuration/index.html#coordinator-operation">indexing period</a>, <code>druid.coordinator.period.indexingPeriod</code>, controls the frequency of compaction tasks.
+The default indexing period is 30 minutes, meaning that the Coordinator first checks for segments to compact at most 30 minutes from when auto-compaction is enabled.
+This time period affects other Coordinator duties including merge and conversion tasks.
+To configure the auto-compaction time period without interfering with <code>indexingPeriod</code>, see <a href="#set-frequency-of-compaction-runs">Set frequency of compaction runs</a>.</p>
+<p>At every invocation of auto-compaction, the Coordinator initiates a <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search</a> to determine eligible segments to compact.
+When there are eligible segments to compact, the Coordinator issues compaction tasks based on available worker capacity.
+If a compaction task takes longer than the indexing period, the Coordinator waits for it to finish before resuming the period for segment search.</p>
+<p>As a best practice, you should set up auto-compaction for all Druid datasources. You can run compaction tasks manually for cases where you want to allocate more system resources. For example, you may choose to run multiple compaction tasks in parallel to compact an existing datasource for the first time. See <a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> for additional details and use cases.</p>
+<p>This topic guides you through setting up automatic compaction for your Druid cluster. See the <a href="#examples">examples</a> for common use cases for automatic compaction.</p>
+<h2><a class="anchor" aria-hidden="true" id="enable-automatic-compaction"></a><a href="#enable-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
+<p>You can enable automatic compaction for a datasource using the web console or programmatically via an API.
+This process differs for manual compaction tasks, which can be submitted from the <a href="/docs/24.0.0/operations/web-console.html">Tasks view of the web console</a> or the <a href="/docs/24.0.0/operations/api-reference.html#post-5">Tasks API</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="web-console"></a><a href="#web-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>Use the web console to enable automatic compaction for a datasource as follows.</p>
+<ol>
+<li>Click <strong>Datasources</strong> in the top-level navigation.</li>
+<li>In the <strong>Compaction</strong> column, click the edit icon for the datasource to compact.</li>
+<li>In the <strong>Compaction config</strong> dialog, configure the auto-compaction settings. The dialog offers a form view as well as a JSON view. Editing the form updates the JSON specification, and editing the JSON updates the form field, if present. Form fields not present in the JSON indicate default values. You may add additional properties to the JSON for auto-compaction settings not displayed in the form. See <a href="#configure-automatic-compaction">Configure automatic compactio [...]
+<li>Click <strong>Submit</strong>.</li>
+<li>Refresh the <strong>Datasources</strong> view. The <strong>Compaction</strong> column for the datasource changes from “Not enabled” to “Awaiting first run.”</li>
+</ol>
+<p>The following screenshot shows the compaction config dialog for a datasource with auto-compaction enabled.
+<img src="../assets/compaction-config.png" alt="Compaction config in web console"></p>
+<p>To disable auto-compaction for a datasource, click <strong>Delete</strong> from the <strong>Compaction config</strong> dialog. Druid does not retain your auto-compaction configuration.</p>
+<h3><a class="anchor" aria-hidden="true" id="compaction-configuration-api"></a><a href="#compaction-configuration-api" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
+<p>Use the <a href="/docs/24.0.0/operations/api-reference.html#automatic-compaction-status">Coordinator API</a> to configure automatic compaction.
+To enable auto-compaction for a datasource, create a JSON object with the desired auto-compaction settings.
+See <a href="#configure-automatic-compaction">Configure automatic compaction</a> for the syntax of an auto-compaction spec.
+Send the JSON object as a payload in a <a href="/docs/24.0.0/operations/api-reference.html#post-4"><code>POST</code> request</a> to <code>/druid/coordinator/v1/config/compaction</code>.
+The following example configures auto-compaction for the <code>wikipedia</code> datasource:</p>
+<pre><code class="hljs css language-sh">curl --location --request POST <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction'</span> \
+--header <span class="hljs-string">'Content-Type: application/json'</span> \
+--data-raw <span class="hljs-string">'{
+ "dataSource": "wikipedia",
+ "granularitySpec": {
+ "segmentGranularity": "DAY"
+ }
+}'</span>
+</code></pre>
+<p>To disable auto-compaction for a datasource, send a <a href="/docs/24.0.0/operations/api-reference.html#delete-1"><code>DELETE</code> request</a> to <code>/druid/coordinator/v1/config/compaction/{dataSource}</code>. Replace <code>{dataSource}</code> with the name of the datasource for which to disable auto-compaction. For example:</p>
+<pre><code class="hljs css language-sh">curl --location --request DELETE <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction/wikipedia'</span>
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="configure-automatic-compaction"></a><a href="#configure-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>You can configure automatic compaction dynamically without restarting Druid.
+The automatic compaction system uses the following syntax:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
+ <span class="hljs-attr">"ioConfig"</span>: <IO config>,
+ <span class="hljs-attr">"dimensionsSpec"</span>: <custom dimensionsSpec>,
+ <span class="hljs-attr">"transformSpec"</span>: <custom transformSpec>,
+ <span class="hljs-attr">"metricsSpec"</span>: <custom metricsSpec>,
+ <span class="hljs-attr">"tuningConfig"</span>: <parallel indexing task tuningConfig>,
+ <span class="hljs-attr">"granularitySpec"</span>: <compaction task granularitySpec>,
+ <span class="hljs-attr">"skipOffsetFromLatest"</span>: <time period to avoid compaction>,
+ <span class="hljs-attr">"taskPriority"</span>: <compaction task priority>,
+ <span class="hljs-attr">"taskContext"</span>: <task context>
+}
+</code></pre>
+<p>Most fields in the auto-compaction configuration correlate to a typical <a href="/docs/24.0.0/ingestion/ingestion-spec.html">Druid ingestion spec</a>.
+The following properties only apply to auto-compaction:</p>
+<ul>
+<li><code>skipOffsetFromLatest</code></li>
+<li><code>taskPriority</code></li>
+<li><code>taskContext</code></li>
+</ul>
+<p>Since the automatic compaction system provides a management layer on top of manual compaction tasks,
+the auto-compaction configuration does not include task-specific properties found in a typical Druid ingestion spec.
+The following properties are automatically set by the Coordinator:</p>
+<ul>
+<li><code>type</code>: Set to <code>compact</code>.</li>
+<li><code>id</code>: Generated using the task type, datasource name, interval, and timestamp. The task ID is prefixed with <code>coordinator-issued</code>.</li>
+<li><code>context</code>: Set according to the user-provided <code>taskContext</code>.</li>
+</ul>
+<p>For more details on each of the specs in an auto-compaction configuration, see <a href="/docs/24.0.0/configuration/index.html#automatic-compaction-dynamic-configuration">Automatic compaction dynamic configuration</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="avoid-conflicts-with-ingestion"></a><a href="#avoid-conflicts-with-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>Compaction tasks may be interrupted when they interfere with ingestion. For example, this occurs when an ingestion task needs to write data to a segment for a time interval locked for compaction. If there are continuous failures that prevent compaction from making progress, consider one of the following strategies:</p>
+<ul>
+<li>Set <code>skipOffsetFromLatest</code> to reduce the chance of conflicts between ingestion and compaction. See more details in this section below.</li>
+<li>Increase the priority value of compaction tasks relative to ingestion tasks. Only recommended for advanced users. This approach can cause ingestion jobs to fail or lag. To change the priority of compaction tasks, set <code>taskPriority</code> to the desired priority value in the auto-compaction configuration. For details on the priority values of different task types, see <a href="/docs/24.0.0/ingestion/tasks.html#lock-priority">Lock priority</a>.</li>
+</ul>
+<p>The Coordinator compacts segments from newest to oldest. In the auto-compaction configuration, you can set a time period, relative to the end time of the most recent segment, for segments that should not be compacted. Assign this value to <code>skipOffsetFromLatest</code>. Note that this offset is not relative to the current time but to the latest segment time. For example, if you want to skip over segments from five days prior to the end time of the most recent segment, assign <code> [...]
+<p>To set <code>skipOffsetFromLatest</code>, consider how frequently you expect the stream to receive late arriving data. If your stream only occasionally receives late arriving data, the auto-compaction system robustly compacts your data even though data is ingested outside the <code>skipOffsetFromLatest</code> window. For most realtime streaming ingestion use cases, it is reasonable to set <code>skipOffsetFromLatest</code> to a few hours or a day.</p>
+<h3><a class="anchor" aria-hidden="true" id="set-frequency-of-compaction-runs"></a><a href="#set-frequency-of-compaction-runs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13. [...]
+<p>If you want the Coordinator to check for compaction more frequently than its indexing period, create a separate group to handle compaction duties.
+Set the time period of the duty group in the <code>coordinator/runtime.properties</code> file.
+The following example shows how to create a duty group named <code>compaction</code> and set the auto-compaction period to 1 minute:</p>
+<pre><code class="hljs"><span class="hljs-attr">druid.coordinator.dutyGroups</span>=[<span class="hljs-string">"compaction"</span>]
+<span class="hljs-attr">druid.coordinator.compaction.duties</span>=[<span class="hljs-string">"compactSegments"</span>]
+<span class="hljs-attr">druid.coordinator.compaction.period</span>=PT60S
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="view-automatic-compaction-statistics"></a><a href="#view-automatic-compaction-statistics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 [...]
+<p>After the Coordinator has initiated auto-compaction, you can view compaction statistics for the datasource, including the number of bytes, segments, and intervals already compacted and those awaiting compaction. The Coordinator also reports the total bytes, segments, and intervals not eligible for compaction in accordance with its <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>.</p>
+<p>In the web console, the Datasources view displays auto-compaction statistics. The Tasks view shows the task information for compaction tasks that were triggered by the automatic compaction system.</p>
+<p>To get statistics by API, send a <a href="/docs/24.0.0/operations/api-reference.html#get-10"><code>GET</code> request</a> to <code>/druid/coordinator/v1/compaction/status</code>. To filter the results to a particular datasource, pass the datasource name as a query parameter to the request—for example, <code>/druid/coordinator/v1/compaction/status?dataSource=wikipedia</code>.</p>
+<h2><a class="anchor" aria-hidden="true" id="examples"></a><a href="#examples" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
+<p>The following examples demonstrate potential use cases in which auto-compaction may improve your Druid performance. See more details in <a href="/docs/24.0.0/data-management/compaction.html#compaction-strategies">Compaction strategies</a>. The examples in this section do not change the underlying data.</p>
+<h3><a class="anchor" aria-hidden="true" id="change-segment-granularity"></a><a href="#change-segment-granularity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
+<p>You have a stream set up to ingest data with <code>HOUR</code> segment granularity into the <code>wikistream</code> datasource. You notice that your Druid segments are smaller than the <a href="/docs/24.0.0/operations/segment-optimization.html">recommended segment size</a> of 5 million rows per segment. You wish to automatically compact segments to <code>DAY</code> granularity while leaving the latest week of data <em>not</em> compacted because your stream consistently receives data w [...]
+<p>The following auto-compaction configuration compacts existing <code>HOUR</code> segments into <code>DAY</code> segments while leaving the latest week of data not compacted:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikistream"</span>,
+ <span class="hljs-attr">"granularitySpec"</span>: {
+ <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"DAY"</span>
+ },
+ <span class="hljs-attr">"skipOffsetFromLatest"</span>: <span class="hljs-string">"P1W"</span>,
+}
+</code></pre>
+<h3><a class="anchor" aria-hidden="true" id="update-partitioning-scheme"></a><a href="#update-partitioning-scheme" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
+<p>For your <code>wikipedia</code> datasource, you want to optimize segment access when regularly ingesting data without compromising compute time when querying the data. Your ingestion spec for batch append uses <a href="/docs/24.0.0/ingestion/native-batch.html#dynamic-partitioning">dynamic partitioning</a> to optimize for write-time operations, while your stream ingestion partitioning is configured by the stream service. You want to implement auto-compaction to reorganize the data with [...]
+<p>The following auto-compaction configuration compacts updates the <code>wikipedia</code> segments to use multi-dimension range partitioning:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
+ <span class="hljs-attr">"tuningConfig"</span>: {
+ <span class="hljs-attr">"partitionsSpec"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"range"</span>,
+ <span class="hljs-attr">"partitionDimensions"</span>: [
+ <span class="hljs-string">"channel"</span>,
+ <span class="hljs-string">"countryName"</span>,
+ <span class="hljs-string">"namespace"</span>
+ ],
+ <span class="hljs-attr">"targetRowsPerSegment"</span>: <span class="hljs-number">5000000</span>
+ }
+ }
+}
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<p>See the following topics for more information:</p>
+<ul>
+<li><a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> for an overview of compaction and how to set up manual compaction in Druid.</li>
+<li><a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for guidance on evaluating and optimizing Druid segment size.</li>
+<li><a href="/docs/24.0.0/design/coordinator.html#automatic-compaction">Coordinator process</a> for details on how the Coordinator plans compaction tasks.</li>
+</ul>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/compaction.html"><span class="arrow-prev">← </span><span>Compaction</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql.html"><span>Overview and syntax</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#enable-automatic-compaction">Enable automatic compaction</a><ul class="toc-head [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/ingestion/compaction.html b/docs/24.0.0/data-management/compaction.html
similarity index 54%
copy from docs/24.0.0/ingestion/compaction.html
copy to docs/24.0.0/data-management/compaction.html
index 8fbfd17b..d404401a 100644
--- a/docs/24.0.0/ingestion/compaction.html
+++ b/docs/24.0.0/data-management/compaction.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/ingestion/compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Use case [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Us [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Compaction</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Compaction</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -97,7 +97,7 @@ Query performance in Apache Druid depends on optimally sized segments. Compactio
<p>Compaction does not improve performance in all situations. For example, if you rewrite your data with each ingestion task, you don't need to use compaction. See <a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for additional guidance to determine if compaction will help in your environment.</p>
<h2><a class="anchor" aria-hidden="true" id="types-of-compaction"></a><a href="#types-of-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
<p>You can configure the Druid Coordinator to perform automatic compaction, also called auto-compaction, for a datasource. Using its <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>, the Coordinator periodically identifies segments for compaction starting from newest to oldest. When the Coordinator discovers segments that have not been compacted or segments that were compacted with a different or changed spec, it submi [...]
-<p>Automatic compaction works in most use cases and should be your first option. To learn more, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a>.</p>
+<p>Automatic compaction works in most use cases and should be your first option. To learn more, see <a href="/docs/24.0.0/data-management/automatic-compaction.html">Automatic compaction</a>.</p>
<p>In cases where you require more control over compaction, you can manually submit compaction tasks. For example:</p>
<ul>
<li>Automatic compaction is running into the limit of task slots available to it, so tasks are waiting for previous automatic compaction tasks to complete. Manual compaction can use all available task slots, therefore you can complete compaction more quickly by submitting more concurrent tasks for more intervals.</li>
@@ -112,7 +112,7 @@ Query performance in Apache Druid depends on optimally sized segments. Compactio
</blockquote>
<p>If an ingestion task needs to write data to a segment for a time interval locked for compaction, by default the ingestion task supersedes the compaction task and the compaction task fails without finishing. For manual compaction tasks, you can adjust the input spec interval to avoid conflicts between ingestion and compaction. For automatic compaction, you can set the <code>skipOffsetFromLatest</code> key to adjust the auto-compaction starting point from the current time to reduce the [...]
Another option is to set the compaction task to higher priority than the ingestion task.
-For more information, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
+For more information, see <a href="/docs/24.0.0/data-management/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="segment-granularity-handling"></a><a href="#segment-granularity-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
<p>Unless you modify the segment granularity in <a href="#compaction-granularity-spec"><code>granularitySpec</code></a>, Druid attempts to retain the granularity for the compacted segments. When segments have different segment granularities with no overlap in interval Druid creates a separate compaction task for each to retain the segment granularity in the compacted segment.</p>
<p>If segments have different segment granularities before compaction but there is some overlap in interval, Druid attempts find start and end of the overlapping interval and uses the closest segment granularity level for the compacted segment.</p>
@@ -171,10 +171,10 @@ You can check that your segments are rolled up or not by using <a href="/docs/24
<blockquote>
<p>You can run multiple compaction tasks in parallel. For example, if you want to compact the data for a year, you are not limited to running a single task for the entire year. You can run 12 compaction tasks with month-long intervals.</p>
</blockquote>
-<p>A compaction task internally generates an <code>index</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">DruidInputSource</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p>
+<p>A compaction task internally generates an <code>index</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>druid</code> input source</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p>
<p>Compaction tasks exit without doing anything and issue a failure status code in either of the following cases:</p>
<ul>
-<li>If the interval you specify has no data segments loaded<br></li>
+<li>If the interval you specify has no data segments loaded<br /></li>
<li>If the interval you specify is empty.</li>
</ul>
<p>Note that the metadata between input segments and the resulting compacted segments may differ if the metadata among the input segments differs as well. If all input segments have the same metadata, however, the resulting output segment will have the same metadata as all input segments.</p>
@@ -207,7 +207,7 @@ If you don't specify <code>granularitySpec</code>, Druid retains the original se
<tbody>
<tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>none</td><td>Yes</td></tr>
<tr><td><code>inputSpec</code></td><td>Specification of the target <a href="#interval-inputspec">intervals</a> or <a href="#segments-inputspec">segments</a>.</td><td>none</td><td>Yes</td></tr>
-<tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br>- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br>- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br>If compaction fails, Druid does not change any of the existing segments.<br><strong>WARNING</strong>: <code>dropExisting</code> in <code>io [...]
+<tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br />- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br />- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br />If compaction fails, Druid does not change any of the existing segments.<br /><strong>WARNING</strong>: <code>dropExisting</code> in [...]
</tbody>
</table>
<p>Druid supports two supported <code>inputSpec</code> formats:</p>
@@ -265,9 +265,9 @@ If you don't specify <code>granularitySpec</code>, Druid retains the original se
<p>See the following topics for more information:</p>
<ul>
<li><a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for guidance to determine if compaction will help in your case.</li>
-<li><a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a> for how to enable and configure automatic compaction.</li>
+<li><a href="/docs/24.0.0/data-management/automatic-compaction.html">Automatic compaction</a> for how to enable and configure automatic compaction.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/data-management.html"><span class="arrow-prev">← </span><span>Data management</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/automatic-compaction.html"><span>Automatic compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#compaction-strategies">Compaction strategies</a></li><li [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/schema-changes.html"><span class="arrow-prev">← </span><span>Schema changes</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/automatic-compaction.html"><span>Automatic compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#compaction-strategies">Compaction strategies</ [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/data-management/delete.html b/docs/24.0.0/data-management/delete.html
new file mode 100644
index 00000000..61823879
--- /dev/null
+++ b/docs/24.0.0/data-management/delete.html
@@ -0,0 +1,156 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data deletion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/delete.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><me [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/delete.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data deletion</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="by-time-range-manually"></a><a href="#by-time-range-manually" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
+<p>Apache Druid stores data <a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> and supports
+deleting data for time chunks by dropping segments. This is a fast, metadata-only operation.</p>
+<p>Deletion by time range happens in two steps:</p>
+<ol>
+<li>Segments to be deleted must first be marked as <a href="/docs/24.0.0/design/architecture.html#segment-lifecycle">"unused"</a>. This can
+happen when a segment is dropped by a <a href="/docs/24.0.0/operations/rule-configuration.html">drop rule</a> or when you manually mark a
+segment unused through the Coordinator API or web console. This is a soft delete: the data is not available for
+querying, but the segment files remains in deep storage, and the segment records remains in the metadata store.</li>
+<li>Once a segment is marked "unused", you can use a <a href="#kill-task"><code>kill</code> task</a> to permanently delete the segment file from
+deep storage and remove its record from the metadata store. This is a hard delete: the data is unrecoverable unless
+you have a backup.</li>
+</ol>
+<p>For documentation on disabling segments using the Coordinator API, see the
+<a href="/docs/24.0.0/operations/api-reference.html#coordinator-datasources">Coordinator API reference</a>.</p>
+<p>A data deletion tutorial is available at <a href="/docs/24.0.0/tutorials/tutorial-delete-data.html">Tutorial: Deleting data</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="by-time-range-automatically"></a><a href="#by-time-range-automatically" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
+<p>Druid supports <a href="/docs/24.0.0/operations/rule-configuration.html">load and drop rules</a>, which are used to define intervals of time
+where data should be preserved, and intervals where data should be discarded. Data that falls under a drop rule is
+marked unused, in the same manner as if you <a href="#by-time-range-manually">manually mark that time range unused</a>. This is a
+fast, metadata-only operation.</p>
+<p>Data that is dropped in this way is marked unused, but remains in deep storage. To permanently delete it, use a
+<a href="#kill-task"><code>kill</code> task</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="specific-records"></a><a href="#specific-records" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
+<p>Druid supports deleting specific records using <a href="/docs/24.0.0/data-management/update.html#reindex">reindexing</a> with a filter. The filter specifies which
+data remains after reindexing, so it must be the inverse of the data you want to delete. Because segments must be
+rewritten to delete data in this way, it can be a time-consuming operation.</p>
+<p>For example, to delete records where <code>userName</code> is <code>'bob'</code> with native batch indexing, use a
+<a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> with filter <code>{"type": "not", "field": {"type": "selector", "dimension": "userName", "value": "bob"}}</code>.</p>
+<p>To delete the same records using SQL, use <a href="/docs/24.0.0/multi-stage-query/concepts.html#replace">REPLACE</a> with <code>WHERE userName <> 'bob'</code>.</p>
+<p>To reindex using <a href="/docs/24.0.0/ingestion/native-batch.html">native batch</a>, use the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#druid-input-source"><code>druid</code> input
+source</a>. If needed,
+<a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> can be used to filter or modify data during the
+reindexing job. To reindex with SQL, use <a href="/docs/24.0.0/multi-stage-query/reference.html#replace"><code>REPLACE <table> OVERWRITE</code></a>
+with <code>SELECT ... FROM <table></code>. (Druid does not have <code>UPDATE</code> or <code>ALTER TABLE</code> statements.) Any SQL SELECT query can be
+used to filter, modify, or enrich the data during the reindexing job.</p>
+<p>Data that is deleted in this way is marked unused, but remains in deep storage. To permanently delete it, use a <a href="#kill-task"><code>kill</code>
+task</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="entire-table"></a><a href="#entire-table" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>Deleting an entire table works the same way as <a href="#by-time-range-manually">deleting part of a table by time range</a>. First,
+mark all segments unused using the Coordinator API or web console. Then, optionally, delete it permanently using a
+<a href="#kill-task"><code>kill</code> task</a>.</p>
+<p><a name="kill-task"></a></p>
+<h2><a class="anchor" aria-hidden="true" id="permanently-kill-task"></a><a href="#permanently-kill-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>Data that has been overwritten or soft-deleted still remains as segments that have been marked unused. You can use a
+<code>kill</code> task to permanently delete this data.</p>
+<p>The available grammar is:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"kill"</span>,
+ <span class="hljs-attr">"id"</span>: <task_id>,
+ <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
+ <span class="hljs-attr">"interval"</span> : <all_unused_segments_in_this_interval_will_die!>,
+ <span class="hljs-attr">"context"</span>: <task context>
+}
+</code></pre>
+<p><strong>WARNING:</strong> The <code>kill</code> task permanently removes all information about the affected segments from the metadata store and
+deep storage. This operation cannot be undone.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/update.html"><span class="arrow-prev">← </span><span>Data updates</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/schema-changes.html"><span>Schema changes</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#by-time-range-manually">By time range, manually</a></li><li><a href= [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/data-management/index.html b/docs/24.0.0/data-management/index.html
new file mode 100644
index 00000000..b9031e16
--- /dev/null
+++ b/docs/24.0.0/data-management/index.html
@@ -0,0 +1,109 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data management · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><m [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data management</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<p>Apache Druid stores data <a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> in immutable
+files called <a href="/docs/24.0.0/design/segments.html">segments</a>. Data management operations involving replacing, or deleting,
+these segments include:</p>
+<ul>
+<li><a href="/docs/24.0.0/data-management/update.html">Updates</a> to existing data.</li>
+<li><a href="/docs/24.0.0/data-management/delete.html">Deletion</a> of existing data.</li>
+<li><a href="/docs/24.0.0/data-management/schema-changes.html">Schema changes</a> for new and existing data.</li>
+<li><a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> and <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a>, which reindex existing data to
+optimize storage footprint and performance.</li>
+</ul>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/faq.html"><span class="arrow-prev">← </span><span>Troubleshooting FAQ</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/update.html"><span>Data updates</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a h [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/data-management/schema-changes.html b/docs/24.0.0/data-management/schema-changes.html
new file mode 100644
index 00000000..c1cf8806
--- /dev/null
+++ b/docs/24.0.0/data-management/schema-changes.html
@@ -0,0 +1,110 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Schema changes · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/schema-changes.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0 [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/schema-changes.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Schema changes</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="for-new-data"></a><a href="#for-new-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>Apache Druid allows you to provide a new schema for new data without the need to update the schema of any existing data.
+It is sufficient to update your supervisor spec, if using <a href="/docs/24.0.0/ingestion/index.html#streaming">streaming ingestion</a>, or to
+provide the new schema the next time you do a <a href="/docs/24.0.0/ingestion/index.html#batch">batch ingestion</a>. This is made possible by
+the fact that each <a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">segment</a>, at the time it is created, stores a
+copy of its own schema. Druid reconciles all of these individual segment schemas automatically at query time.</p>
+<h2><a class="anchor" aria-hidden="true" id="for-existing-data"></a><a href="#for-existing-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
+<p>Schema changes are sometimes necessary for existing data. For example, you may want to change the type of a column in
+previously-ingested data, or drop a column entirely. Druid handles this using <a href="/docs/24.0.0/data-management/update.html">reindexing</a>, the same method
+it uses to handle updates of existing data. Reindexing involves rewriting all affected segments and can be a
+time-consuming operation.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/delete.html"><span class="arrow-prev">← </span><span>Data deletion</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/compaction.html"><span>Compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#for-new-data">For new data</a></li><li><a href="#for-existing-data">For exi [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/data-management/update.html b/docs/24.0.0/data-management/update.html
new file mode 100644
index 00000000..fe111793
--- /dev/null
+++ b/docs/24.0.0/data-management/update.html
@@ -0,0 +1,138 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data updates · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/data-management/update.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><met [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/update.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data updates</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="overwrite"></a><a href="#overwrite" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
+<p>Apache Druid stores data <a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> and supports
+overwriting existing data using time ranges. Data outside the replacement time range is not touched. Overwriting of
+existing data is done using the same mechanisms as <a href="/docs/24.0.0/ingestion/index.html#batch">batch ingestion</a>.</p>
+<p>For example:</p>
+<ul>
+<li><a href="/docs/24.0.0/ingestion/native-batch.html">Native batch</a> with <code>appendToExisting: false</code>, and <code>intervals</code> set to a specific
+time range, overwrites data for that time range.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/reference.html#replace">SQL <code>REPLACE <table> OVERWRITE [ALL | WHERE ...]</code></a> overwrites data for
+the entire table or for a specified time range.</li>
+</ul>
+<p>In both cases, Druid's atomic update mechanism ensures that queries will flip seamlessly from the old data to the new
+data on a time-chunk-by-time-chunk basis.</p>
+<p>Ingestion and overwriting cannot run concurrently for the same time range of the same datasource. While an overwrite job
+is ongoing for a particular time range of a datasource, new ingestions for that time range are queued up. Ingestions for
+other time ranges proceed as normal. Read-only queries also proceed as normal, using the pre-existing version of the
+data.</p>
+<p>Druid does not support single-record updates by primary key.</p>
+<h2><a class="anchor" aria-hidden="true" id="reindex"></a><a href="#reindex" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<p>Reindexing is an <a href="#overwrite">overwrite of existing data</a> where the source of new data is the existing data itself. It
+is used to perform schema changes, repartition data, filter out unwanted data, enrich existing data, and so on. This
+behaves just like any other <a href="#overwrite">overwrite</a> with regard to atomic updates and locking.</p>
+<p>With <a href="/docs/24.0.0/ingestion/native-batch.html">native batch</a>, use the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#druid-input-source"><code>druid</code> input
+source</a>. If needed,
+<a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> can be used to filter or modify data during the
+reindexing job.</p>
+<p>With SQL, use <a href="/docs/24.0.0/multi-stage-query/reference.html#replace"><code>REPLACE <table> OVERWRITE</code></a> with `SELECT ... FROM</p>
+<table>`. (Druid does not have `UPDATE` or `ALTER TABLE` statements.) Any SQL SELECT query can be used to filter,
+modify, or enrich the data during the reindexing job.
+<h2><a class="anchor" aria-hidden="true" id="rolled-up-datasources"></a><a href="#rolled-up-datasources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>Rolled-up datasources can be effectively updated using appends, without rewrites. When you append a row that has an
+identical set of dimensions to an existing row, queries that use aggregation operators automatically combine those two
+rows together at query time.</p>
+<p><a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> or <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a> can be used to physically combine these
+matching rows together later on, by rewriting segments in the background.</p>
+<h2><a class="anchor" aria-hidden="true" id="lookups"></a><a href="#lookups" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<p>If you have a dimension where values need to be updated frequently, try first using <a href="/docs/24.0.0/querying/lookups.html">lookups</a>. A
+classic use case of lookups is when you have an ID dimension stored in a Druid segment, and want to map the ID dimension to a
+human-readable string that may need to be updated periodically.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/index.html"><span class="arrow-prev">← </span><span>Overview</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/delete.html"><span>Data deletion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#overwrite">Overwrite</a></li><li><a href="#reindex">Reindex</a></li><li><a href="# [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/dependencies/deep-storage.html b/docs/24.0.0/dependencies/deep-storage.html
index ad2b87ca..1a5305a8 100644
--- a/docs/24.0.0/dependencies/deep-storage.html
+++ b/docs/24.0.0/dependencies/deep-storage.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/dependencies/metadata-storage.html b/docs/24.0.0/dependencies/metadata-storage.html
index 0833eeb0..af2c6ff0 100644
--- a/docs/24.0.0/dependencies/metadata-storage.html
+++ b/docs/24.0.0/dependencies/metadata-storage.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/dependencies/zookeeper.html b/docs/24.0.0/dependencies/zookeeper.html
index 7d941cb6..cafb04e1 100644
--- a/docs/24.0.0/dependencies/zookeeper.html
+++ b/docs/24.0.0/dependencies/zookeeper.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/architecture.html b/docs/24.0.0/design/architecture.html
index b75929df..81e6ed06 100644
--- a/docs/24.0.0/design/architecture.html
+++ b/docs/24.0.0/design/architecture.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -92,8 +92,8 @@
<li><a href="/docs/24.0.0/design/historical.html"><strong>Historical</strong></a> services store queryable data.</li>
<li><a href="/docs/24.0.0/design/middlemanager.html"><strong>MiddleManager</strong></a> services ingest data.</li>
</ul>
-<p>You can view services in the <strong>Services</strong> tab in the Druid console:</p>
-<p><img src="../assets/services-overview.png" alt="Druid services" title="Services in the Druid console"></p>
+<p>You can view services in the <strong>Services</strong> tab in the web console:</p>
+<p><img src="../assets/services-overview.png" alt="Druid services" title="Services in the web console"></p>
<h2><a class="anchor" aria-hidden="true" id="druid-servers"></a><a href="#druid-servers" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
<p>Druid services can be deployed any way you like, but for ease of deployment we suggest organizing them into three server types: Master, Query, and Data.</p>
<ul>
@@ -317,7 +317,7 @@ query.</p>
</ul>
<p>For more details about how Druid executes queries, refer to the <a href="/docs/24.0.0/querying/query-execution.html">Query execution</a>
documentation.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-msq-convert-json.html"><span class="arrow-prev">← </span><span>Convert ingestion spec to SQL</span></a><a class="docs-next button" href="/docs/24.0.0/design/segments.html"><span>Segments</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#druid-architecture">Druid architecture</a></li><li><a href= [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html"><span class="arrow-prev">← </span><span>Convert ingestion spec to SQL</span></a><a class="docs-next button" href="/docs/24.0.0/design/segments.html"><span>Segments</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#druid-architecture">Druid architecture</a></li><li><a href= [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/design/auth.html b/docs/24.0.0/design/auth.html
index 2fa1af4e..24670cb7 100644
--- a/docs/24.0.0/design/auth.html
+++ b/docs/24.0.0/design/auth.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/broker.html b/docs/24.0.0/design/broker.html
index 7e096c81..2748bfad 100644
--- a/docs/24.0.0/design/broker.html
+++ b/docs/24.0.0/design/broker.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/coordinator.html b/docs/24.0.0/design/coordinator.html
index df1d33ad..f7ccc403 100644
--- a/docs/24.0.0/design/coordinator.html
+++ b/docs/24.0.0/design/coordinator.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -115,7 +115,7 @@ marked as unused. During the next Coordinator's run, they will be unloaded from
<h3><a class="anchor" aria-hidden="true" id="balancing-segment-load"></a><a href="#balancing-segment-load" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
<p>To ensure an even distribution of segments across Historical processes in the cluster, the Coordinator process will find the total size of all segments being served by every Historical process each time the Coordinator runs. For every Historical process tier in the cluster, the Coordinator process will determine the Historical process with the highest utilization and the Historical process with the lowest utilization. The percent difference in utilization between the two processes is [...]
<h3><a class="anchor" aria-hidden="true" id="automatic-compaction"></a><a href="#automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1 [...]
-<p>The Druid Coordinator manages the <a href="/docs/24.0.0/ingestion/automatic-compaction.html">automatic compaction system</a>.
+<p>The Druid Coordinator manages the <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction system</a>.
Each run, the Coordinator compacts segments by merging small segments or splitting a large one. This is useful when the size of your segments is not optimized which may degrade query performance.
See <a href="/docs/24.0.0/operations/segment-optimization.html">Segment size optimization</a> for details.</p>
<p>The Coordinator first finds the segments to compact based on the <a href="#segment-search-policy-in-automatic-compaction">segment search policy</a>.
@@ -167,7 +167,7 @@ Finally, <code>foo_2017-09-01T00:00:00.000Z_2017-10-01T00:00:00.000Z_VERSION</co
If this is set, this policy will ignore the segments falling into the time chunk of (the end time of the most recent segment - <code>skipOffsetFromLatest</code>).
This is to avoid conflicts between compaction tasks and realtime tasks.
Note that realtime tasks have a higher priority than compaction tasks by default. Realtime tasks will revoke the locks of compaction tasks if their intervals overlap, resulting in the termination of the compaction task.
-For more information, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
+For more information, see <a href="/docs/24.0.0/data-management/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
<blockquote>
<p>This policy currently cannot handle the situation when there are a lot of small segments which have the same interval,
and their total size exceeds <a href="/docs/24.0.0/configuration/index.html#automatic-compaction-dynamic-configuration"><code>inputSegmentSizeBytes</code></a>.
diff --git a/docs/24.0.0/design/historical.html b/docs/24.0.0/design/historical.html
index 1ad99fcb..3f344ea0 100644
--- a/docs/24.0.0/design/historical.html
+++ b/docs/24.0.0/design/historical.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/index.html b/docs/24.0.0/design/index.html
index 1a4567ed..b82348b2 100644
--- a/docs/24.0.0/design/index.html
+++ b/docs/24.0.0/design/index.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -145,7 +145,7 @@ taking a long time to complete.</li>
<li>Learn more about Druid components in <a href="/docs/24.0.0/design/architecture.html">Design</a>.</li>
<li>Read about new features and other details of <a href="https://github.com/apache/druid/releases">Druid Releases</a>.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/24.0.0/tutorials/index.html"><span>Quickstart</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#key-features-of-druid">Key features of Druid</a></li><li><a href="#when-to-use-druid">When to use Druid</a></li><li><a href="#learn-more">Learn more</a></li></ul></nav></div><footer class="nav-footer druid-footer" id="footer"> [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-next button" href="/docs/24.0.0/tutorials/index.html"><span>Quickstart (local)</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#key-features-of-druid">Key features of Druid</a></li><li><a href="#when-to-use-druid">When to use Druid</a></li><li><a href="#learn-more">Learn more</a></li></ul></nav></div><footer class="nav-footer druid-footer" id=" [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/design/indexer.html b/docs/24.0.0/design/indexer.html
index c45f79f4..8a084cae 100644
--- a/docs/24.0.0/design/indexer.html
+++ b/docs/24.0.0/design/indexer.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -114,7 +114,6 @@ Its memory management system is still under development and will be significantl
<h3><a class="anchor" aria-hidden="true" id="current-limitations"></a><a href="#current-limitations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
<p>Separate task logs are not currently supported when using the Indexer; all task log messages will instead be logged in the Indexer process log.</p>
<p>The Indexer currently imposes an identical memory limit on each task. In later releases, the per-task memory limit will be removed and only the global limit will apply. The limit on concurrent merges will also be removed.</p>
-<p>The Indexer does not work properly with <a href="/docs/24.0.0/ingestion/tasks.html#index_realtime"><code>index_realtime</code></a> task types. Therefore, it is not compatible with <a href="/docs/24.0.0/ingestion/tranquility.html">Tranquility</a>. If you are using Tranquility, consider migrating to Druid's builtin <a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Apache Kafka</a> or <a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Amazon K [...]
<p>In later releases, per-task memory usage will be dynamically managed. Please see <a href="https://github.com/apache/druid/issues/7900">https://github.com/apache/druid/issues/7900</a> for details on future enhancements to the Indexer.</p>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/design/historical.html"><span class="arrow-prev">← </span><span>Historical Process</span></a><a class="docs-next button" href="/docs/24.0.0/design/indexing-service.html"><span>Indexing Service</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center" [...]
document.addEventListener('keyup', function(e) {
diff --git a/docs/24.0.0/design/indexing-service.html b/docs/24.0.0/design/indexing-service.html
index c7264915..86679ec6 100644
--- a/docs/24.0.0/design/indexing-service.html
+++ b/docs/24.0.0/design/indexing-service.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/middlemanager.html b/docs/24.0.0/design/middlemanager.html
index 2e712f64..a4b89c2b 100644
--- a/docs/24.0.0/design/middlemanager.html
+++ b/docs/24.0.0/design/middlemanager.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/overlord.html b/docs/24.0.0/design/overlord.html
index ed4a54a7..45c0909f 100644
--- a/docs/24.0.0/design/overlord.html
+++ b/docs/24.0.0/design/overlord.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/peons.html b/docs/24.0.0/design/peons.html
index bc19020b..119fcd7c 100644
--- a/docs/24.0.0/design/peons.html
+++ b/docs/24.0.0/design/peons.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/design/processes.html b/docs/24.0.0/design/processes.html
index ceecb329..86ba4172 100644
--- a/docs/24.0.0/design/processes.html
+++ b/docs/24.0.0/design/processes.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -114,7 +114,7 @@ caller. End users typically query Brokers rather than querying Historicals or Mi
<p><a href="/docs/24.0.0/design/router.html"><strong>Router</strong></a> processes are <em>optional</em> processes that provide a unified API gateway in front of Druid Brokers,
Overlords, and Coordinators. They are optional since you can also simply contact the Druid Brokers, Overlords, and
Coordinators directly.</p>
-<p>The Router also runs the <a href="/docs/24.0.0/operations/druid-console.html">Druid console</a>, a management UI for datasources, segments, tasks, data processes (Historicals and MiddleManagers), and coordinator dynamic configuration. The user can also run SQL and native Druid queries within the console.</p>
+<p>The Router also runs the <a href="/docs/24.0.0/operations/web-console.html">web console</a>, a management UI for datasources, segments, tasks, data processes (Historicals and MiddleManagers), and coordinator dynamic configuration. The user can also run SQL and native Druid queries within the console.</p>
<h3><a class="anchor" aria-hidden="true" id="data-server"></a><a href="#data-server" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
<p>A Data server executes ingestion jobs and stores queryable data.</p>
<p>Within a Data server, functionality is split between two processes, the Historical and MiddleManager.</p>
diff --git a/docs/24.0.0/design/router.html b/docs/24.0.0/design/router.html
index 41362e57..a73935a0 100644
--- a/docs/24.0.0/design/router.html
+++ b/docs/24.0.0/design/router.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -79,7 +79,7 @@
-->
<p>The Apache Druid Router process can be used to route queries to different Broker processes. By default, the broker routes queries based on how <a href="/docs/24.0.0/operations/rule-configuration.html">Rules</a> are set up. For example, if 1 month of recent data is loaded into a <code>hot</code> cluster, queries that fall within the recent month can be routed to a dedicated set of brokers. Queries outside this range are routed to another set of brokers. This set up provides query isola [...]
<p>For query routing purposes, you should only ever need the Router process if you have a Druid cluster well into the terabyte range.</p>
-<p>In addition to query routing, the Router also runs the <a href="/docs/24.0.0/operations/druid-console.html">Druid console</a>, a management UI for datasources, segments, tasks, data processes (Historicals and MiddleManagers), and coordinator dynamic configuration. The user can also run SQL and native Druid queries within the console.</p>
+<p>In addition to query routing, the Router also runs the <a href="/docs/24.0.0/operations/web-console.html">web console</a>, a management UI for datasources, segments, tasks, data processes (Historicals and MiddleManagers), and coordinator dynamic configuration. The user can also run SQL and native Druid queries within the console.</p>
<h3><a class="anchor" aria-hidden="true" id="configuration"></a><a href="#configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
<p>For Apache Druid Router Process Configuration, see <a href="/docs/24.0.0/configuration/index.html#router">Router Configuration</a>.</p>
<p>For basic tuning guidance for the Router process, see <a href="/docs/24.0.0/operations/basic-cluster-tuning.html#router">Basic cluster tuning</a>.</p>
diff --git a/docs/24.0.0/design/segments.html b/docs/24.0.0/design/segments.html
index 38ea1bdd..34591afe 100644
--- a/docs/24.0.0/design/segments.html
+++ b/docs/24.0.0/design/segments.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -77,7 +77,7 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<p>Apache Druid stores its data and indexes in <em>segment files</em> partitioned by time. Druid creates a segment for each segment interval that contains data. If an interval is empty—that is, containing no rows—no segment exists for that time interval. Druid may create multiple segments for the same interval if you ingest data for that period via different ingestion jobs. <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a> is the Druid process that attempts to combine these [...]
+<p>Apache Druid stores its data and indexes in <em>segment files</em> partitioned by time. Druid creates a segment for each segment interval that contains data. If an interval is empty—that is, containing no rows—no segment exists for that time interval. Druid may create multiple segments for the same interval if you ingest data for that period via different ingestion jobs. <a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> is the Druid process that attempts to combine [...]
<p>The time interval is configurable in the <code>segmentGranularity</code> parameter of the <a href="/docs/24.0.0/ingestion/ingestion-spec.html#granularityspec"><code>granularitySpec</code></a>.</p>
<p>For Druid to operate well under heavy query load, it is important for the segment
file size to be within the recommended range of 300-700 MB. If your
diff --git a/docs/24.0.0/development/build.html b/docs/24.0.0/development/build.html
index f9153301..cf5e04d4 100644
--- a/docs/24.0.0/development/build.html
+++ b/docs/24.0.0/development/build.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/experimental.html b/docs/24.0.0/development/experimental.html
index 444c6eb8..55d47674 100644
--- a/docs/24.0.0/development/experimental.html
+++ b/docs/24.0.0/development/experimental.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/aliyun-oss.html b/docs/24.0.0/development/extensions-contrib/aliyun-oss.html
index 563212e5..0f453092 100644
--- a/docs/24.0.0/development/extensions-contrib/aliyun-oss.html
+++ b/docs/24.0.0/development/extensions-contrib/aliyun-oss.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/ambari-metrics-emitter.html b/docs/24.0.0/development/extensions-contrib/ambari-metrics-emitter.html
index 207167f0..aadffe5b 100644
--- a/docs/24.0.0/development/extensions-contrib/ambari-metrics-emitter.html
+++ b/docs/24.0.0/development/extensions-contrib/ambari-metrics-emitter.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/cassandra.html b/docs/24.0.0/development/extensions-contrib/cassandra.html
index 2a424fc7..f1e7bb80 100644
--- a/docs/24.0.0/development/extensions-contrib/cassandra.html
+++ b/docs/24.0.0/development/extensions-contrib/cassandra.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/cloudfiles.html b/docs/24.0.0/development/extensions-contrib/cloudfiles.html
index 6cc25784..5c3eb644 100644
--- a/docs/24.0.0/development/extensions-contrib/cloudfiles.html
+++ b/docs/24.0.0/development/extensions-contrib/cloudfiles.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/distinctcount.html b/docs/24.0.0/development/extensions-contrib/distinctcount.html
index e5b39744..693189bd 100644
--- a/docs/24.0.0/development/extensions-contrib/distinctcount.html
+++ b/docs/24.0.0/development/extensions-contrib/distinctcount.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/gce-extensions.html b/docs/24.0.0/development/extensions-contrib/gce-extensions.html
index 6c755c38..da78a9df 100644
--- a/docs/24.0.0/development/extensions-contrib/gce-extensions.html
+++ b/docs/24.0.0/development/extensions-contrib/gce-extensions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/graphite.html b/docs/24.0.0/development/extensions-contrib/graphite.html
index 21692e92..bf691130 100644
--- a/docs/24.0.0/development/extensions-contrib/graphite.html
+++ b/docs/24.0.0/development/extensions-contrib/graphite.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/influx.html b/docs/24.0.0/development/extensions-contrib/influx.html
index c9b69742..0dd5ab54 100644
--- a/docs/24.0.0/development/extensions-contrib/influx.html
+++ b/docs/24.0.0/development/extensions-contrib/influx.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/influxdb-emitter.html b/docs/24.0.0/development/extensions-contrib/influxdb-emitter.html
index df396073..57c63091 100644
--- a/docs/24.0.0/development/extensions-contrib/influxdb-emitter.html
+++ b/docs/24.0.0/development/extensions-contrib/influxdb-emitter.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/kafka-emitter.html b/docs/24.0.0/development/extensions-contrib/kafka-emitter.html
index 6bb4386f..6f50a59d 100644
--- a/docs/24.0.0/development/extensions-contrib/kafka-emitter.html
+++ b/docs/24.0.0/development/extensions-contrib/kafka-emitter.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -79,7 +79,7 @@
-->
<p>To use this Apache Druid extension, <a href="/docs/24.0.0/development/extensions.html#loading-extensions">include</a> <code>kafka-emitter</code> in the extensions load list.</p>
<h2><a class="anchor" aria-hidden="true" id="introduction"></a><a href="#introduction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
-<p>This extension emits Druid metrics to <a href="https://kafka.apache.org">Apache Kafka</a> directly with JSON format.<br>
+<p>This extension emits Druid metrics to <a href="https://kafka.apache.org">Apache Kafka</a> directly with JSON format.<br />
Currently, Kafka has not only their nice ecosystem but also consumer API readily available.
So, If you currently use Kafka, It's easy to integrate various tool or UI
to monitor the status of your Druid cluster with this extension.</p>
diff --git a/docs/24.0.0/development/extensions-contrib/materialized-view.html b/docs/24.0.0/development/extensions-contrib/materialized-view.html
index df9e37da..d3ad8b44 100644
--- a/docs/24.0.0/development/extensions-contrib/materialized-view.html
+++ b/docs/24.0.0/development/extensions-contrib/materialized-view.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/momentsketch-quantiles.html b/docs/24.0.0/development/extensions-contrib/momentsketch-quantiles.html
index 82535df5..8c728863 100644
--- a/docs/24.0.0/development/extensions-contrib/momentsketch-quantiles.html
+++ b/docs/24.0.0/development/extensions-contrib/momentsketch-quantiles.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/moving-average-query.html b/docs/24.0.0/development/extensions-contrib/moving-average-query.html
index 36e94420..09762657 100644
--- a/docs/24.0.0/development/extensions-contrib/moving-average-query.html
+++ b/docs/24.0.0/development/extensions-contrib/moving-average-query.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/opentsdb-emitter.html b/docs/24.0.0/development/extensions-contrib/opentsdb-emitter.html
index 144d4147..892d8023 100644
--- a/docs/24.0.0/development/extensions-contrib/opentsdb-emitter.html
+++ b/docs/24.0.0/development/extensions-contrib/opentsdb-emitter.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/prometheus.html b/docs/24.0.0/development/extensions-contrib/prometheus.html
index ca5c68a1..d3ad7fc1 100644
--- a/docs/24.0.0/development/extensions-contrib/prometheus.html
+++ b/docs/24.0.0/development/extensions-contrib/prometheus.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/redis-cache.html b/docs/24.0.0/development/extensions-contrib/redis-cache.html
index 1e578652..e9bf0751 100644
--- a/docs/24.0.0/development/extensions-contrib/redis-cache.html
+++ b/docs/24.0.0/development/extensions-contrib/redis-cache.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/sqlserver.html b/docs/24.0.0/development/extensions-contrib/sqlserver.html
index 7baafcae..a9d952ae 100644
--- a/docs/24.0.0/development/extensions-contrib/sqlserver.html
+++ b/docs/24.0.0/development/extensions-contrib/sqlserver.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/statsd.html b/docs/24.0.0/development/extensions-contrib/statsd.html
index cc27f7fb..b206b4b8 100644
--- a/docs/24.0.0/development/extensions-contrib/statsd.html
+++ b/docs/24.0.0/development/extensions-contrib/statsd.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/tdigestsketch-quantiles.html b/docs/24.0.0/development/extensions-contrib/tdigestsketch-quantiles.html
index 66ad5d0d..e86a20c8 100644
--- a/docs/24.0.0/development/extensions-contrib/tdigestsketch-quantiles.html
+++ b/docs/24.0.0/development/extensions-contrib/tdigestsketch-quantiles.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/thrift.html b/docs/24.0.0/development/extensions-contrib/thrift.html
index d5cd117a..f02b40d0 100644
--- a/docs/24.0.0/development/extensions-contrib/thrift.html
+++ b/docs/24.0.0/development/extensions-contrib/thrift.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-contrib/time-min-max.html b/docs/24.0.0/development/extensions-contrib/time-min-max.html
index e8d303c4..893149d8 100644
--- a/docs/24.0.0/development/extensions-contrib/time-min-max.html
+++ b/docs/24.0.0/development/extensions-contrib/time-min-max.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/approximate-histograms.html b/docs/24.0.0/development/extensions-core/approximate-histograms.html
index 5cf86c9b..f0e410c6 100644
--- a/docs/24.0.0/development/extensions-core/approximate-histograms.html
+++ b/docs/24.0.0/development/extensions-core/approximate-histograms.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/avro.html b/docs/24.0.0/development/extensions-core/avro.html
index ef62d00c..cdf41be2 100644
--- a/docs/24.0.0/development/extensions-core/avro.html
+++ b/docs/24.0.0/development/extensions-core/avro.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/azure.html b/docs/24.0.0/development/extensions-core/azure.html
index c46abc81..df3e7a20 100644
--- a/docs/24.0.0/development/extensions-core/azure.html
+++ b/docs/24.0.0/development/extensions-core/azure.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/bloom-filter.html b/docs/24.0.0/development/extensions-core/bloom-filter.html
index d3e3b4fa..5b76e400 100644
--- a/docs/24.0.0/development/extensions-core/bloom-filter.html
+++ b/docs/24.0.0/development/extensions-core/bloom-filter.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/datasketches-extension.html b/docs/24.0.0/development/extensions-core/datasketches-extension.html
index fcef333a..c92eb79f 100644
--- a/docs/24.0.0/development/extensions-core/datasketches-extension.html
+++ b/docs/24.0.0/development/extensions-core/datasketches-extension.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/datasketches-hll.html b/docs/24.0.0/development/extensions-core/datasketches-hll.html
index d06f6ca3..34656928 100644
--- a/docs/24.0.0/development/extensions-core/datasketches-hll.html
+++ b/docs/24.0.0/development/extensions-core/datasketches-hll.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/datasketches-quantiles.html b/docs/24.0.0/development/extensions-core/datasketches-quantiles.html
index 1cb015b6..206b31a2 100644
--- a/docs/24.0.0/development/extensions-core/datasketches-quantiles.html
+++ b/docs/24.0.0/development/extensions-core/datasketches-quantiles.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/datasketches-theta.html b/docs/24.0.0/development/extensions-core/datasketches-theta.html
index e0cdd727..f3dbea29 100644
--- a/docs/24.0.0/development/extensions-core/datasketches-theta.html
+++ b/docs/24.0.0/development/extensions-core/datasketches-theta.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/datasketches-tuple.html b/docs/24.0.0/development/extensions-core/datasketches-tuple.html
index 21021e7c..c3ba249a 100644
--- a/docs/24.0.0/development/extensions-core/datasketches-tuple.html
+++ b/docs/24.0.0/development/extensions-core/datasketches-tuple.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/druid-aws-rds.html b/docs/24.0.0/development/extensions-core/druid-aws-rds.html
index 44bf0a9a..8d15b6ca 100644
--- a/docs/24.0.0/development/extensions-core/druid-aws-rds.html
+++ b/docs/24.0.0/development/extensions-core/druid-aws-rds.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/druid-basic-security.html b/docs/24.0.0/development/extensions-core/druid-basic-security.html
index bacf888e..f1427458 100644
--- a/docs/24.0.0/development/extensions-core/druid-basic-security.html
+++ b/docs/24.0.0/development/extensions-core/druid-basic-security.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/druid-kerberos.html b/docs/24.0.0/development/extensions-core/druid-kerberos.html
index 83c2e79b..95e781cd 100644
--- a/docs/24.0.0/development/extensions-core/druid-kerberos.html
+++ b/docs/24.0.0/development/extensions-core/druid-kerberos.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/druid-lookups.html b/docs/24.0.0/development/extensions-core/druid-lookups.html
index 7617c385..58f3a306 100644
--- a/docs/24.0.0/development/extensions-core/druid-lookups.html
+++ b/docs/24.0.0/development/extensions-core/druid-lookups.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/druid-pac4j.html b/docs/24.0.0/development/extensions-core/druid-pac4j.html
index 01dd22de..8ee1ce3f 100644
--- a/docs/24.0.0/development/extensions-core/druid-pac4j.html
+++ b/docs/24.0.0/development/extensions-core/druid-pac4j.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -79,7 +79,7 @@
-->
<p>Apache Druid Extension to enable <a href="https://openid.net/connect/">OpenID Connect</a> based Authentication for Druid Processes using <a href="https://github.com/pac4j/pac4j">pac4j</a> as the underlying client library.
This can be used with any authentication server that supports same e.g. <a href="https://developer.okta.com/">Okta</a>.
-This extension should only be used at the router node to enable a group of users in existing authentication server to interact with Druid cluster, using the <a href="/docs/24.0.0/operations/druid-console.html">Druid console</a>. This extension does not support JDBC client authentication.</p>
+This extension should only be used at the router node to enable a group of users in existing authentication server to interact with Druid cluster, using the <a href="/docs/24.0.0/operations/web-console.html">web console</a>. This extension does not support JDBC client authentication.</p>
<h2><a class="anchor" aria-hidden="true" id="configuration"></a><a href="#configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
<h3><a class="anchor" aria-hidden="true" id="creating-an-authenticator"></a><a href="#creating-an-authenticator" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c [...]
<pre><code class="hljs"><span class="hljs-attr">druid.auth.authenticatorChain</span>=[<span class="hljs-string">"pac4j"</span>]
diff --git a/docs/24.0.0/development/extensions-core/druid-ranger-security.html b/docs/24.0.0/development/extensions-core/druid-ranger-security.html
index 1d1b65b9..a2e7b9bb 100644
--- a/docs/24.0.0/development/extensions-core/druid-ranger-security.html
+++ b/docs/24.0.0/development/extensions-core/druid-ranger-security.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/google.html b/docs/24.0.0/development/extensions-core/google.html
index 360cf2ad..016aa8fa 100644
--- a/docs/24.0.0/development/extensions-core/google.html
+++ b/docs/24.0.0/development/extensions-core/google.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/hdfs.html b/docs/24.0.0/development/extensions-core/hdfs.html
index b318541d..0fd00ea7 100644
--- a/docs/24.0.0/development/extensions-core/hdfs.html
+++ b/docs/24.0.0/development/extensions-core/hdfs.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/kafka-extraction-namespace.html b/docs/24.0.0/development/extensions-core/kafka-extraction-namespace.html
index 3832f3d5..d61d2010 100644
--- a/docs/24.0.0/development/extensions-core/kafka-extraction-namespace.html
+++ b/docs/24.0.0/development/extensions-core/kafka-extraction-namespace.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/kafka-ingestion.html b/docs/24.0.0/development/extensions-core/kafka-ingestion.html
index b3c23332..7fd2d1f2 100644
--- a/docs/24.0.0/development/extensions-core/kafka-ingestion.html
+++ b/docs/24.0.0/development/extensions-core/kafka-ingestion.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -297,13 +297,13 @@ Additionally, you must provide information about how Druid should parse the data
</code></pre>
<p>For more information, see <a href="/docs/24.0.0/ingestion/data-formats.html#kafka"><code>kafka</code> data format</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="submit-a-supervisor-spec"></a><a href="#submit-a-supervisor-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-. [...]
-<p>Druid starts a supervisor for a dataSource when you submit a supervisor spec. You can use the data loader in the Druid console or you can submit a supervisor spec to the following endpoint:</p>
+<p>Druid starts a supervisor for a dataSource when you submit a supervisor spec. You can use the data loader in the web console or you can submit a supervisor spec to the following endpoint:</p>
<p><code>http://<OVERLORD_IP>:<OVERLORD_PORT>/druid/indexer/v1/supervisor</code></p>
<p>For example:</p>
<pre><code class="hljs">curl -X POST -H <span class="hljs-string">'Content-Type: application/json'</span> -d @supervisor-spec.json http:<span class="hljs-regexp">//</span>localhost:<span class="hljs-number">8090</span><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>supervisor
</code></pre>
<p>Where the file <code>supervisor-spec.json</code> contains your Kafka supervisor spec file.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/automatic-compaction.html"><span class="arrow-prev">← </span><span>Automatic compaction</span></a><a class="docs-next button" href="/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html"><span>Apache Kafka supervisor</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#kafka-support">Kafk [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/schema-design.html"><span class="arrow-prev">← </span><span>Schema design tips</span></a><a class="docs-next button" href="/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html"><span>Apache Kafka supervisor</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#kafka-support">Kafka support [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/development/extensions-core/kafka-supervisor-operations.html b/docs/24.0.0/development/extensions-core/kafka-supervisor-operations.html
index 69f84d58..4b0d96da 100644
--- a/docs/24.0.0/development/extensions-core/kafka-supervisor-operations.html
+++ b/docs/24.0.0/development/extensions-core/kafka-supervisor-operations.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html b/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html
index 1bfdbd8b..ef619bb9 100644
--- a/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html
+++ b/docs/24.0.0/development/extensions-core/kafka-supervisor-reference.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -210,7 +210,7 @@ By default, <code>isolation.level</code> is set to <code>read_committed</code>.
}
}
</code></pre>
-<p>Verify that you've changed the values for all configurations to match your own environment. You can use the environment variable config provider syntax in the <strong>Consumer properties</strong> field on the <strong>Connect tab</strong> in the <strong>Load Data</strong> UI in the Druid console. When connecting to Kafka, Druid replaces the environment variables with their corresponding values.</p>
+<p>Verify that you've changed the values for all configurations to match your own environment. You can use the environment variable config provider syntax in the <strong>Consumer properties</strong> field on the <strong>Connect tab</strong> in the <strong>Load Data</strong> UI in the web console. When connecting to Kafka, Druid replaces the environment variables with their corresponding values.</p>
<p>Note: You can provide SSL connections with <a href="/docs/24.0.0/operations/password-provider.html">Password Provider</a> interface to define the <code>keystore</code>, <code>truststore</code>, and <code>key</code>, but this feature is deprecated.</p>
<h2><a class="anchor" aria-hidden="true" id="specifying-data-format"></a><a href="#specifying-data-format" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
<p>Kafka indexing service supports both <a href="/docs/24.0.0/ingestion/data-formats.html#input-format"><code>inputFormat</code></a> and <a href="/docs/24.0.0/ingestion/data-formats.html#parser"><code>parser</code></a> to specify the data format.
diff --git a/docs/24.0.0/development/extensions-core/kinesis-ingestion.html b/docs/24.0.0/development/extensions-core/kinesis-ingestion.html
index 473f1afc..7dca9cb3 100644
--- a/docs/24.0.0/development/extensions-core/kinesis-ingestion.html
+++ b/docs/24.0.0/development/extensions-core/kinesis-ingestion.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/kubernetes.html b/docs/24.0.0/development/extensions-core/kubernetes.html
index f5e58a75..b5f87f0b 100644
--- a/docs/24.0.0/development/extensions-core/kubernetes.html
+++ b/docs/24.0.0/development/extensions-core/kubernetes.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/lookups-cached-global.html b/docs/24.0.0/development/extensions-core/lookups-cached-global.html
index b110b9c6..af142368 100644
--- a/docs/24.0.0/development/extensions-core/lookups-cached-global.html
+++ b/docs/24.0.0/development/extensions-core/lookups-cached-global.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/mysql.html b/docs/24.0.0/development/extensions-core/mysql.html
index 463255ad..1f8a2f01 100644
--- a/docs/24.0.0/development/extensions-core/mysql.html
+++ b/docs/24.0.0/development/extensions-core/mysql.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/orc.html b/docs/24.0.0/development/extensions-core/orc.html
index 57b138f0..27656675 100644
--- a/docs/24.0.0/development/extensions-core/orc.html
+++ b/docs/24.0.0/development/extensions-core/orc.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/parquet.html b/docs/24.0.0/development/extensions-core/parquet.html
index aa22a76a..5aa6a0b5 100644
--- a/docs/24.0.0/development/extensions-core/parquet.html
+++ b/docs/24.0.0/development/extensions-core/parquet.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/postgresql.html b/docs/24.0.0/development/extensions-core/postgresql.html
index 206d410e..678d8f78 100644
--- a/docs/24.0.0/development/extensions-core/postgresql.html
+++ b/docs/24.0.0/development/extensions-core/postgresql.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/protobuf.html b/docs/24.0.0/development/extensions-core/protobuf.html
index 6eb0a7ac..fa014c68 100644
--- a/docs/24.0.0/development/extensions-core/protobuf.html
+++ b/docs/24.0.0/development/extensions-core/protobuf.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/s3.html b/docs/24.0.0/development/extensions-core/s3.html
index 492dee9f..0bb596c1 100644
--- a/docs/24.0.0/development/extensions-core/s3.html
+++ b/docs/24.0.0/development/extensions-core/s3.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/simple-client-sslcontext.html b/docs/24.0.0/development/extensions-core/simple-client-sslcontext.html
index 2748af9d..140cda04 100644
--- a/docs/24.0.0/development/extensions-core/simple-client-sslcontext.html
+++ b/docs/24.0.0/development/extensions-core/simple-client-sslcontext.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/stats.html b/docs/24.0.0/development/extensions-core/stats.html
index 22d98e9c..ced6efc7 100644
--- a/docs/24.0.0/development/extensions-core/stats.html
+++ b/docs/24.0.0/development/extensions-core/stats.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions-core/test-stats.html b/docs/24.0.0/development/extensions-core/test-stats.html
index a2218678..f889a8fd 100644
--- a/docs/24.0.0/development/extensions-core/test-stats.html
+++ b/docs/24.0.0/development/extensions-core/test-stats.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/extensions.html b/docs/24.0.0/development/extensions.html
index 71a17697..1548f585 100644
--- a/docs/24.0.0/development/extensions.html
+++ b/docs/24.0.0/development/extensions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/geo.html b/docs/24.0.0/development/geo.html
index 7658399c..e4bcd487 100644
--- a/docs/24.0.0/development/geo.html
+++ b/docs/24.0.0/development/geo.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/javascript.html b/docs/24.0.0/development/javascript.html
index b160a3cf..4035e834 100644
--- a/docs/24.0.0/development/javascript.html
+++ b/docs/24.0.0/development/javascript.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/modules.html b/docs/24.0.0/development/modules.html
index 2dec2000..48e511af 100644
--- a/docs/24.0.0/development/modules.html
+++ b/docs/24.0.0/development/modules.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -171,7 +171,7 @@ T00:00:00.000Z/2015<span class="hljs-string">-04</span><span class="hljs-string"
<ul>
<li>DataSegmentKiller</li>
</ul>
-<p>The easiest way of testing the segment killing is marking a segment as not used and then starting a killing task in the <a href="/docs/24.0.0/operations/druid-console.html">web console</a>.</p>
+<p>The easiest way of testing the segment killing is marking a segment as not used and then starting a killing task in the <a href="/docs/24.0.0/operations/web-console.html">web console</a>.</p>
<p>To mark a segment as not used, you need to connect to your metadata storage and update the <code>used</code> column to <code>false</code> on the segment table rows.</p>
<p>To start a segment killing task, you need to access the web console then select <code>issue kill task</code> for the appropriate datasource.</p>
<p>After the killing task ends, <code>index.zip</code> (<code>partitionNum_index.zip</code> for HDFS data storage) file should be deleted from the data storage.</p>
diff --git a/docs/24.0.0/development/overview.html b/docs/24.0.0/development/overview.html
index 4001439b..b00518a6 100644
--- a/docs/24.0.0/development/overview.html
+++ b/docs/24.0.0/development/overview.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/development/versioning.html b/docs/24.0.0/development/versioning.html
index fe8d281d..eea33139 100644
--- a/docs/24.0.0/development/versioning.html
+++ b/docs/24.0.0/development/versioning.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/ingestion/automatic-compaction.html b/docs/24.0.0/ingestion/automatic-compaction.html
index 28e66604..7d466702 100644
--- a/docs/24.0.0/ingestion/automatic-compaction.html
+++ b/docs/24.0.0/ingestion/automatic-compaction.html
@@ -1,232 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Automatic compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/ingestion/automatic-compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/automatic-compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Automatic compaction</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<p>In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to <a href="/docs/24.0.0/operations/segment-optimization.html">optimally size segments</a> after ingestion to improve query performance. Automatic compaction, or auto-compaction, refers to the system for automatic execution of compaction tasks managed by the <a href="/docs/24.0.0/design/coordinator.html">Drui [...]
-<p>The Coordinator <a href="/docs/24.0.0/configuration/index.html#coordinator-operation">indexing period</a>, <code>druid.coordinator.period.indexingPeriod</code>, controls the frequency of compaction tasks.
-The default indexing period is 30 minutes, meaning that the Coordinator first checks for segments to compact at most 30 minutes from when auto-compaction is enabled.
-This time period affects other Coordinator duties including merge and conversion tasks.
-To configure the auto-compaction time period without interfering with <code>indexingPeriod</code>, see <a href="#set-frequency-of-compaction-runs">Set frequency of compaction runs</a>.</p>
-<p>At every invocation of auto-compaction, the Coordinator initiates a <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search</a> to determine eligible segments to compact.
-When there are eligible segments to compact, the Coordinator issues compaction tasks based on available worker capacity.
-If a compaction task takes longer than the indexing period, the Coordinator waits for it to finish before resuming the period for segment search.</p>
-<p>As a best practice, you should set up auto-compaction for all Druid datasources. You can run compaction tasks manually for cases where you want to allocate more system resources. For example, you may choose to run multiple compaction tasks in parallel to compact an existing datasource for the first time. See <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a> for additional details and use cases.</p>
-<p>This topic guides you through setting up automatic compaction for your Druid cluster. See the <a href="#examples">examples</a> for common use cases for automatic compaction.</p>
-<h2><a class="anchor" aria-hidden="true" id="enable-automatic-compaction"></a><a href="#enable-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
-<p>You can enable automatic compaction for a datasource using the Druid console or programmatically via an API.
-This process differs for manual compaction tasks, which can be submitted from the <a href="/docs/24.0.0/operations/druid-console.html">Tasks view of the Druid console</a> or the <a href="/docs/24.0.0/operations/api-reference.html#post-5">Tasks API</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="druid-console"></a><a href="#druid-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
-<p>Use the Druid console to enable automatic compaction for a datasource as follows.</p>
-<ol>
-<li>Click <strong>Datasources</strong> in the top-level navigation.</li>
-<li>In the <strong>Compaction</strong> column, click the edit icon for the datasource to compact.</li>
-<li>In the <strong>Compaction config</strong> dialog, configure the auto-compaction settings. The dialog offers a form view as well as a JSON view. Editing the form updates the JSON specification, and editing the JSON updates the form field, if present. Form fields not present in the JSON indicate default values. You may add additional properties to the JSON for auto-compaction settings not displayed in the form. See <a href="#configure-automatic-compaction">Configure automatic compactio [...]
-<li>Click <strong>Submit</strong>.</li>
-<li>Refresh the <strong>Datasources</strong> view. The <strong>Compaction</strong> column for the datasource changes from “Not enabled” to “Awaiting first run.”</li>
-</ol>
-<p>The following screenshot shows the compaction config dialog for a datasource with auto-compaction enabled.
-<img src="../assets/compaction-config.png" alt="Compaction config in web console"></p>
-<p>To disable auto-compaction for a datasource, click <strong>Delete</strong> from the <strong>Compaction config</strong> dialog. Druid does not retain your auto-compaction configuration.</p>
-<h3><a class="anchor" aria-hidden="true" id="compaction-configuration-api"></a><a href="#compaction-configuration-api" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
-<p>Use the <a href="/docs/24.0.0/operations/api-reference.html#automatic-compaction-status">Coordinator API</a> to configure automatic compaction.
-To enable auto-compaction for a datasource, create a JSON object with the desired auto-compaction settings.
-See <a href="#configure-automatic-compaction">Configure automatic compaction</a> for the syntax of an auto-compaction spec.
-Send the JSON object as a payload in a <a href="/docs/24.0.0/operations/api-reference.html#post-4"><code>POST</code> request</a> to <code>/druid/coordinator/v1/config/compaction</code>.
-The following example configures auto-compaction for the <code>wikipedia</code> datasource:</p>
-<pre><code class="hljs css language-sh">curl --location --request POST <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction'</span> \
---header <span class="hljs-string">'Content-Type: application/json'</span> \
---data-raw <span class="hljs-string">'{
- "dataSource": "wikipedia",
- "granularitySpec": {
- "segmentGranularity": "DAY"
- }
-}'</span>
-</code></pre>
-<p>To disable auto-compaction for a datasource, send a <a href="/docs/24.0.0/operations/api-reference.html#delete-1"><code>DELETE</code> request</a> to <code>/druid/coordinator/v1/config/compaction/{dataSource}</code>. Replace <code>{dataSource}</code> with the name of the datasource for which to disable auto-compaction. For example:</p>
-<pre><code class="hljs css language-sh">curl --location --request DELETE <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction/wikipedia'</span>
-</code></pre>
-<h2><a class="anchor" aria-hidden="true" id="configure-automatic-compaction"></a><a href="#configure-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
-<p>You can configure automatic compaction dynamically without restarting Druid.
-The automatic compaction system uses the following syntax:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
- <span class="hljs-attr">"ioConfig"</span>: <IO config>,
- <span class="hljs-attr">"dimensionsSpec"</span>: <custom dimensionsSpec>,
- <span class="hljs-attr">"transformSpec"</span>: <custom transformSpec>,
- <span class="hljs-attr">"metricsSpec"</span>: <custom metricsSpec>,
- <span class="hljs-attr">"tuningConfig"</span>: <parallel indexing task tuningConfig>,
- <span class="hljs-attr">"granularitySpec"</span>: <compaction task granularitySpec>,
- <span class="hljs-attr">"skipOffsetFromLatest"</span>: <time period to avoid compaction>,
- <span class="hljs-attr">"taskPriority"</span>: <compaction task priority>,
- <span class="hljs-attr">"taskContext"</span>: <task context>
-}
-</code></pre>
-<p>Most fields in the auto-compaction configuration correlate to a typical <a href="/docs/24.0.0/ingestion/ingestion-spec.html">Druid ingestion spec</a>.
-The following properties only apply to auto-compaction:</p>
-<ul>
-<li><code>skipOffsetFromLatest</code></li>
-<li><code>taskPriority</code></li>
-<li><code>taskContext</code></li>
-</ul>
-<p>Since the automatic compaction system provides a management layer on top of manual compaction tasks,
-the auto-compaction configuration does not include task-specific properties found in a typical Druid ingestion spec.
-The following properties are automatically set by the Coordinator:</p>
-<ul>
-<li><code>type</code>: Set to <code>compact</code>.</li>
-<li><code>id</code>: Generated using the task type, datasource name, interval, and timestamp. The task ID is prefixed with <code>coordinator-issued</code>.</li>
-<li><code>context</code>: Set according to the user-provided <code>taskContext</code>.</li>
-</ul>
-<p>For more details on each of the specs in an auto-compaction configuration, see <a href="/docs/24.0.0/configuration/index.html#automatic-compaction-dynamic-configuration">Automatic compaction dynamic configuration</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="avoid-conflicts-with-ingestion"></a><a href="#avoid-conflicts-with-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
-<p>Compaction tasks may be interrupted when they interfere with ingestion. For example, this occurs when an ingestion task needs to write data to a segment for a time interval locked for compaction. If there are continuous failures that prevent compaction from making progress, consider one of the following strategies:</p>
-<ul>
-<li>Set <code>skipOffsetFromLatest</code> to reduce the chance of conflicts between ingestion and compaction. See more details in this section below.</li>
-<li>Increase the priority value of compaction tasks relative to ingestion tasks. Only recommended for advanced users. This approach can cause ingestion jobs to fail or lag. To change the priority of compaction tasks, set <code>taskPriority</code> to the desired priority value in the auto-compaction configuration. For details on the priority values of different task types, see <a href="/docs/24.0.0/ingestion/tasks.html#lock-priority">Lock priority</a>.</li>
-</ul>
-<p>The Coordinator compacts segments from newest to oldest. In the auto-compaction configuration, you can set a time period, relative to the end time of the most recent segment, for segments that should not be compacted. Assign this value to <code>skipOffsetFromLatest</code>. Note that this offset is not relative to the current time but to the latest segment time. For example, if you want to skip over segments from five days prior to the end time of the most recent segment, assign <code> [...]
-<p>To set <code>skipOffsetFromLatest</code>, consider how frequently you expect the stream to receive late arriving data. If your stream only occasionally receives late arriving data, the auto-compaction system robustly compacts your data even though data is ingested outside the <code>skipOffsetFromLatest</code> window. For most realtime streaming ingestion use cases, it is reasonable to set <code>skipOffsetFromLatest</code> to a few hours or a day.</p>
-<h3><a class="anchor" aria-hidden="true" id="set-frequency-of-compaction-runs"></a><a href="#set-frequency-of-compaction-runs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13. [...]
-<p>If you want the Coordinator to check for compaction more frequently than its indexing period, create a separate group to handle compaction duties.
-Set the time period of the duty group in the <code>coordinator/runtime.properties</code> file.
-The following example shows how to create a duty group named <code>compaction</code> and set the auto-compaction period to 1 minute:</p>
-<pre><code class="hljs"><span class="hljs-attr">druid.coordinator.dutyGroups</span>=[<span class="hljs-string">"compaction"</span>]
-<span class="hljs-attr">druid.coordinator.compaction.duties</span>=[<span class="hljs-string">"compactSegments"</span>]
-<span class="hljs-attr">druid.coordinator.compaction.period</span>=PT60S
-</code></pre>
-<h2><a class="anchor" aria-hidden="true" id="view-automatic-compaction-statistics"></a><a href="#view-automatic-compaction-statistics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 [...]
-<p>After the Coordinator has initiated auto-compaction, you can view compaction statistics for the datasource, including the number of bytes, segments, and intervals already compacted and those awaiting compaction. The Coordinator also reports the total bytes, segments, and intervals not eligible for compaction in accordance with its <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>.</p>
-<p>In the Druid console, the Datasources view displays auto-compaction statistics. The Tasks view shows the task information for compaction tasks that were triggered by the automatic compaction system.</p>
-<p>To get statistics by API, send a <a href="/docs/24.0.0/operations/api-reference.html#get-10"><code>GET</code> request</a> to <code>/druid/coordinator/v1/compaction/status</code>. To filter the results to a particular datasource, pass the datasource name as a query parameter to the request—for example, <code>/druid/coordinator/v1/compaction/status?dataSource=wikipedia</code>.</p>
-<h2><a class="anchor" aria-hidden="true" id="examples"></a><a href="#examples" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
-<p>The following examples demonstrate potential use cases in which auto-compaction may improve your Druid performance. See more details in <a href="/docs/24.0.0/ingestion/compaction.html#compaction-strategies">Compaction strategies</a>. The examples in this section do not change the underlying data.</p>
-<h3><a class="anchor" aria-hidden="true" id="change-segment-granularity"></a><a href="#change-segment-granularity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
-<p>You have a stream set up to ingest data with <code>HOUR</code> segment granularity into the <code>wikistream</code> datasource. You notice that your Druid segments are smaller than the <a href="/docs/24.0.0/operations/segment-optimization.html">recommended segment size</a> of 5 million rows per segment. You wish to automatically compact segments to <code>DAY</code> granularity while leaving the latest week of data <em>not</em> compacted because your stream consistently receives data w [...]
-<p>The following auto-compaction configuration compacts existing <code>HOUR</code> segments into <code>DAY</code> segments while leaving the latest week of data not compacted:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikistream"</span>,
- <span class="hljs-attr">"granularitySpec"</span>: {
- <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"DAY"</span>
- },
- <span class="hljs-attr">"skipOffsetFromLatest"</span>: <span class="hljs-string">"P1W"</span>,
-}
-</code></pre>
-<h3><a class="anchor" aria-hidden="true" id="update-partitioning-scheme"></a><a href="#update-partitioning-scheme" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
-<p>For your <code>wikipedia</code> datasource, you want to optimize segment access when regularly ingesting data without compromising compute time when querying the data. Your ingestion spec for batch append uses <a href="/docs/24.0.0/ingestion/native-batch.html#dynamic-partitioning">dynamic partitioning</a> to optimize for write-time operations, while your stream ingestion partitioning is configured by the stream service. You want to implement auto-compaction to reorganize the data with [...]
-<p>The following auto-compaction configuration compacts updates the <code>wikipedia</code> segments to use multi-dimension range partitioning:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
- <span class="hljs-attr">"tuningConfig"</span>: {
- <span class="hljs-attr">"partitionsSpec"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"range"</span>,
- <span class="hljs-attr">"partitionDimensions"</span>: [
- <span class="hljs-string">"channel"</span>,
- <span class="hljs-string">"countryName"</span>,
- <span class="hljs-string">"namespace"</span>
- ],
- <span class="hljs-attr">"targetRowsPerSegment"</span>: <span class="hljs-number">5000000</span>
- }
- }
-}
-</code></pre>
-<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<p>See the following topics for more information:</p>
-<ul>
-<li><a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a> for an overview of compaction and how to set up manual compaction in Druid.</li>
-<li><a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for guidance on evaluating and optimizing Druid segment size.</li>
-<li><a href="/docs/24.0.0/design/coordinator.html#automatic-compaction">Coordinator process</a> for details on how the Coordinator plans compaction tasks.</li>
-</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/compaction.html"><span class="arrow-prev">← </span><span>Compaction</span></a><a class="docs-next button" href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html"><span>Apache Kafka ingestion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#enable-automatic-compaction">Enable automatic compa [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="../data-management/automatic-compaction.html">
+<meta http-equiv="refresh" content="0; url=../data-management/automatic-compaction.html">
+<h1>Redirecting...</h1>
+<a href="../data-management/automatic-compaction.html">Click here if you are not redirected.</a>
+<script>location="../data-management/automatic-compaction.html"</script>
diff --git a/docs/24.0.0/ingestion/compaction.html b/docs/24.0.0/ingestion/compaction.html
index 8fbfd17b..83b43d5e 100644
--- a/docs/24.0.0/ingestion/compaction.html
+++ b/docs/24.0.0/ingestion/compaction.html
@@ -1,289 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/ingestion/compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Use case [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Compaction</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-Query performance in Apache Druid depends on optimally sized segments. Compaction is one strategy you can use to optimize segment size for your Druid database. Compaction tasks read an existing set of segments for a given time interval and combine the data into a new "compacted" set of segments. In some cases the compacted segments are larger, but there are fewer of them. In other cases the compacted segments may be smaller. Compaction tends to increase performance because optimized segm [...]
-<h2><a class="anchor" aria-hidden="true" id="compaction-strategies"></a><a href="#compaction-strategies" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
-<p>There are several cases to consider compaction for segment optimization:</p>
-<ul>
-<li>With streaming ingestion, data can arrive out of chronological order creating many small segments.</li>
-<li>If you append data using <code>appendToExisting</code> for <a href="/docs/24.0.0/ingestion/native-batch.html">native batch</a> ingestion creating suboptimal segments.</li>
-<li>When you use <code>index_parallel</code> for parallel batch indexing and the parallel ingestion tasks create many small segments.</li>
-<li>When a misconfigured ingestion task creates oversized segments.</li>
-</ul>
-<p>By default, compaction does not modify the underlying data of the segments. However, there are cases when you may want to modify data during compaction to improve query performance:</p>
-<ul>
-<li>If, after ingestion, you realize that data for the time interval is sparse, you can use compaction to increase the segment granularity.</li>
-<li>If you don't need fine-grained granularity for older data, you can use compaction to change older segments to a coarser query granularity. For example, from <code>minute</code> to <code>hour</code> or <code>hour</code> to <code>day</code>. This reduces the storage space required for older data.</li>
-<li>You can change the dimension order to improve sorting and reduce segment size.</li>
-<li>You can remove unused columns in compaction or implement an aggregation metric for older data.</li>
-<li>You can change segment rollup from dynamic partitioning with best-effort rollup to hash or range partitioning with perfect rollup. For more information on rollup, see <a href="/docs/24.0.0/ingestion/rollup.html#perfect-rollup-vs-best-effort-rollup">perfect vs best-effort rollup</a>.</li>
-</ul>
-<p>Compaction does not improve performance in all situations. For example, if you rewrite your data with each ingestion task, you don't need to use compaction. See <a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for additional guidance to determine if compaction will help in your environment.</p>
-<h2><a class="anchor" aria-hidden="true" id="types-of-compaction"></a><a href="#types-of-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
-<p>You can configure the Druid Coordinator to perform automatic compaction, also called auto-compaction, for a datasource. Using its <a href="/docs/24.0.0/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>, the Coordinator periodically identifies segments for compaction starting from newest to oldest. When the Coordinator discovers segments that have not been compacted or segments that were compacted with a different or changed spec, it submi [...]
-<p>Automatic compaction works in most use cases and should be your first option. To learn more, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a>.</p>
-<p>In cases where you require more control over compaction, you can manually submit compaction tasks. For example:</p>
-<ul>
-<li>Automatic compaction is running into the limit of task slots available to it, so tasks are waiting for previous automatic compaction tasks to complete. Manual compaction can use all available task slots, therefore you can complete compaction more quickly by submitting more concurrent tasks for more intervals.</li>
-<li>You want to force compaction for a specific time range or you want to compact data out of chronological order.</li>
-</ul>
-<p>See <a href="#setting-up-manual-compaction">Setting up a manual compaction task</a> for more about manual compaction tasks.</p>
-<h2><a class="anchor" aria-hidden="true" id="data-handling-with-compaction"></a><a href="#data-handling-with-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 [...]
-<p>During compaction, Druid overwrites the original set of segments with the compacted set. Druid also locks the segments for the time interval being compacted to ensure data consistency. By default, compaction tasks do not modify the underlying data. You can configure the compaction task to change the query granularity or add or remove dimensions in the compaction task. This means that the only changes to query results should be the result of intentional, not automatic, changes.</p>
-<p>You can set <code>dropExisting</code> in <code>ioConfig</code> to "true" in the compaction task to configure Druid to replace all existing segments fully contained by the interval. See the suggestion for reindexing with finer granularity under <a href="/docs/24.0.0/ingestion/native-batch.html#implementation-considerations">Implementation considerations</a> for an example.</p>
-<blockquote>
-<p>WARNING: <code>dropExisting</code> in <code>ioConfig</code> is a beta feature.</p>
-</blockquote>
-<p>If an ingestion task needs to write data to a segment for a time interval locked for compaction, by default the ingestion task supersedes the compaction task and the compaction task fails without finishing. For manual compaction tasks, you can adjust the input spec interval to avoid conflicts between ingestion and compaction. For automatic compaction, you can set the <code>skipOffsetFromLatest</code> key to adjust the auto-compaction starting point from the current time to reduce the [...]
-Another option is to set the compaction task to higher priority than the ingestion task.
-For more information, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="segment-granularity-handling"></a><a href="#segment-granularity-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
-<p>Unless you modify the segment granularity in <a href="#compaction-granularity-spec"><code>granularitySpec</code></a>, Druid attempts to retain the granularity for the compacted segments. When segments have different segment granularities with no overlap in interval Druid creates a separate compaction task for each to retain the segment granularity in the compacted segment.</p>
-<p>If segments have different segment granularities before compaction but there is some overlap in interval, Druid attempts find start and end of the overlapping interval and uses the closest segment granularity level for the compacted segment.</p>
-<p>For example consider two overlapping segments: segment "A" for the interval 01/01/2021-01/02/2021 with day granularity and segment "B" for the interval 01/01/2021-02/01/2021. Druid attempts to combine and compact the overlapped segments. In this example, the earliest start time for the two segments is 01/01/2020 and the latest end time of the two segments is 02/01/2020. Druid compacts the segments together even though they have different segment granularity. Druid [...]
-<h3><a class="anchor" aria-hidden="true" id="query-granularity-handling"></a><a href="#query-granularity-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
-<p>Unless you modify the query granularity in the <a href="#compaction-granularity-spec"><code>granularitySpec</code></a>, Druid retains the query granularity for the compacted segments. If segments have different query granularities before compaction, Druid chooses the finest level of granularity for the resulting compacted segment. For example if a compaction task combines two segments, one with day query granularity and one with minute query granularity, the resulting segment uses min [...]
-<blockquote>
-<p>In Apache Druid 0.21.0 and prior, Druid sets the granularity for compacted segments to the default granularity of <code>NONE</code> regardless of the query granularity of the original segments.</p>
-</blockquote>
-<p>If you configure query granularity in compaction to go from a finer granularity like month to a coarser query granularity like year, then Druid overshadows the original segment with coarser granularity. Because the new segments have a coarser granularity, running a kill task to remove the overshadowed segments for those intervals will cause you to permanently lose the finer granularity data.</p>
-<h3><a class="anchor" aria-hidden="true" id="dimension-handling"></a><a href="#dimension-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>Apache Druid supports schema changes. Therefore, dimensions can be different across segments even if they are a part of the same datasource. See <a href="/docs/24.0.0/design/segments.html#segments-with-different-schemas">Segments with different schemas</a>. If the input segments have different dimensions, the resulting compacted segment includes all dimensions of the input segments.</p>
-<p>Even when the input segments have the same set of dimensions, the dimension order or the data type of dimensions can be different. The dimensions of recent segments precede that of old segments in terms of data types and the ordering because more recent segments are more likely to have the preferred order and data types.</p>
-<p>If you want to control dimension ordering or ensure specific values for dimension types, you can configure a custom <code>dimensionsSpec</code> in the compaction task spec.</p>
-<h3><a class="anchor" aria-hidden="true" id="rollup"></a><a href="#rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
-<p>Druid only rolls up the output segment when <code>rollup</code> is set for all input segments.
-See <a href="/docs/24.0.0/ingestion/rollup.html">Roll-up</a> for more details.
-You can check that your segments are rolled up or not by using <a href="/docs/24.0.0/querying/segmentmetadataquery.html#analysistypes">Segment Metadata Queries</a>.</p>
-<h2><a class="anchor" aria-hidden="true" id="setting-up-manual-compaction"></a><a href="#setting-up-manual-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
-<p>To perform a manual compaction, you submit a compaction task. Compaction tasks merge all segments for the defined interval according to the following syntax:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"compact"</span>,
- <span class="hljs-attr">"id"</span>: <task_id>,
- <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
- <span class="hljs-attr">"ioConfig"</span>: <IO config>,
- <span class="hljs-attr">"dimensionsSpec"</span>: <custom dimensionsSpec>,
- <span class="hljs-attr">"transformSpec"</span>: <custom transformSpec>,
- <span class="hljs-attr">"metricsSpec"</span>: <custom metricsSpec>,
- <span class="hljs-attr">"tuningConfig"</span>: <parallel indexing task tuningConfig>,
- <span class="hljs-attr">"granularitySpec"</span>: <compaction task granularitySpec>,
- <span class="hljs-attr">"context"</span>: <task context>
-}
-</code></pre>
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>Yes</td></tr>
-<tr><td><code>id</code></td><td>Task ID</td><td>No</td></tr>
-<tr><td><code>dataSource</code></td><td>Data source name to compact</td><td>Yes</td></tr>
-<tr><td><code>ioConfig</code></td><td>I/O configuration for compaction task. See <a href="#compaction-io-configuration">Compaction I/O configuration</a> for details.</td><td>Yes</td></tr>
-<tr><td><code>dimensionsSpec</code></td><td>When set, the compaction task uses the specified <code>dimensionsSpec</code> instead of generating one. See <a href="#compaction-dimensions-spec">Compaction dimensionsSpec</a> for details.</td><td>No</td></tr>
-<tr><td><code>transformSpec</code></td><td>When set, the compaction task uses the specified <code>transformSpec</code> rather than using <code>null</code>. See <a href="#compaction-transform-spec">Compaction transformSpec</a> for details.</td><td>No</td></tr>
-<tr><td><code>metricsSpec</code></td><td>When set, the compaction task uses the specified <code>metricsSpec</code> rather than generating one.</td><td>No</td></tr>
-<tr><td><code>segmentGranularity</code></td><td>Deprecated. Use <code>granularitySpec</code>.</td><td>No</td></tr>
-<tr><td><code>tuningConfig</code></td><td><a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig">Tuning configuration</a> for parallel indexing. <code>awaitSegmentAvailabilityTimeoutMillis</code> value is not supported for compaction tasks. Leave this parameter at the default value, 0.</td><td>No</td></tr>
-<tr><td><code>granularitySpec</code></td><td>When set, the compaction task uses the specified <code>granularitySpec</code> rather than generating one. See <a href="#compaction-granularity-spec">Compaction <code>granularitySpec</code></a> for details.</td><td>No</td></tr>
-<tr><td><code>context</code></td><td><a href="/docs/24.0.0/ingestion/tasks.html#context">Task context</a></td><td>No</td></tr>
-</tbody>
-</table>
-<blockquote>
-<p>Note: Use <code>granularitySpec</code> over <code>segmentGranularity</code> and only set one of these values. If you specify different values for these in the same compaction spec, the task fails.</p>
-</blockquote>
-<p>To control the number of result segments per time chunk, you can set <a href="/docs/24.0.0/ingestion/native-batch.html#partitionsspec"><code>maxRowsPerSegment</code></a> or <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig"><code>numShards</code></a>.</p>
-<blockquote>
-<p>You can run multiple compaction tasks in parallel. For example, if you want to compact the data for a year, you are not limited to running a single task for the entire year. You can run 12 compaction tasks with month-long intervals.</p>
-</blockquote>
-<p>A compaction task internally generates an <code>index</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">DruidInputSource</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p>
-<p>Compaction tasks exit without doing anything and issue a failure status code in either of the following cases:</p>
-<ul>
-<li>If the interval you specify has no data segments loaded<br></li>
-<li>If the interval you specify is empty.</li>
-</ul>
-<p>Note that the metadata between input segments and the resulting compacted segments may differ if the metadata among the input segments differs as well. If all input segments have the same metadata, however, the resulting output segment will have the same metadata as all input segments.</p>
-<h3><a class="anchor" aria-hidden="true" id="example-compaction-task"></a><a href="#example-compaction-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 [...]
-<p>The following JSON illustrates a compaction task to compact <em>all segments</em> within the interval <code>2020-01-01/2021-01-01</code> and create new segments:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"compact"</span>,
- <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
- <span class="hljs-attr">"ioConfig"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"compact"</span>,
- <span class="hljs-attr">"inputSpec"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"interval"</span>,
- <span class="hljs-attr">"interval"</span>: <span class="hljs-string">"2020-01-01/2021-01-01"</span>
- }
- },
- <span class="hljs-attr">"granularitySpec"</span>: {
- <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"day"</span>,
- <span class="hljs-attr">"queryGranularity"</span>: <span class="hljs-string">"hour"</span>
- }
-}
-</code></pre>
-<p><code>granularitySpec</code> is an optional field.
-If you don't specify <code>granularitySpec</code>, Druid retains the original segment and query granularities when compaction is complete.</p>
-<h3><a class="anchor" aria-hidden="true" id="compaction-io-configuration"></a><a href="#compaction-io-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
-<p>The compaction <code>ioConfig</code> requires specifying <code>inputSpec</code> as follows:</p>
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Default</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>none</td><td>Yes</td></tr>
-<tr><td><code>inputSpec</code></td><td>Specification of the target <a href="#interval-inputspec">intervals</a> or <a href="#segments-inputspec">segments</a>.</td><td>none</td><td>Yes</td></tr>
-<tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br>- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br>- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br>If compaction fails, Druid does not change any of the existing segments.<br><strong>WARNING</strong>: <code>dropExisting</code> in <code>io [...]
-</tbody>
-</table>
-<p>Druid supports two supported <code>inputSpec</code> formats:</p>
-<h4><a class="anchor" aria-hidden="true" id="interval-inputspec"></a><a href="#interval-inputspec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>type</code></td><td>Task type. Set the value to <code>interval</code>.</td><td>Yes</td></tr>
-<tr><td><code>interval</code></td><td>Interval to compact.</td><td>Yes</td></tr>
-</tbody>
-</table>
-<h4><a class="anchor" aria-hidden="true" id="segments-inputspec"></a><a href="#segments-inputspec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>type</code></td><td>Task type. Set the value to <code>segments</code>.</td><td>Yes</td></tr>
-<tr><td><code>segments</code></td><td>A list of segment IDs.</td><td>Yes</td></tr>
-</tbody>
-</table>
-<h3><a class="anchor" aria-hidden="true" id="compaction-dimensions-spec"></a><a href="#compaction-dimensions-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>dimensions</code></td><td>A list of dimension names or objects. Cannot have the same column in both <code>dimensions</code> and <code>dimensionExclusions</code>. Defaults to <code>null</code>, which preserves the original dimensions.</td><td>No</td></tr>
-<tr><td><code>dimensionExclusions</code></td><td>The names of dimensions to exclude from compaction. Only names are supported here, not objects. This list is only used if the dimensions list is null or empty; otherwise it is ignored. Defaults to <code>[]</code>.</td><td>No</td></tr>
-</tbody>
-</table>
-<h3><a class="anchor" aria-hidden="true" id="compaction-transform-spec"></a><a href="#compaction-transform-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>filter</code></td><td>The <code>filter</code> conditionally filters input rows during compaction. Only rows that pass the filter will be included in the compacted segments. Any of Druid's standard <a href="/docs/24.0.0/querying/filters.html">query filters</a> can be used. Defaults to 'null', which will not filter any row.</td><td>No</td></tr>
-</tbody>
-</table>
-<h3><a class="anchor" aria-hidden="true" id="compaction-granularity-spec"></a><a href="#compaction-granularity-spec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td><code>segmentGranularity</code></td><td>Time chunking period for the segment granularity. Defaults to 'null', which preserves the original segment granularity. Accepts all <a href="/docs/24.0.0/querying/granularities.html">Query granularity</a> values.</td><td>No</td></tr>
-<tr><td><code>queryGranularity</code></td><td>The resolution of timestamp storage within each segment. Defaults to 'null', which preserves the original query granularity. Accepts all <a href="/docs/24.0.0/querying/granularities.html">Query granularity</a> values.</td><td>No</td></tr>
-<tr><td><code>rollup</code></td><td>Enables compaction-time rollup. To preserve the original setting, keep the default value. To enable compaction-time rollup, set the value to <code>true</code>. Once the data is rolled up, you can no longer recover individual records.</td><td>No</td></tr>
-</tbody>
-</table>
-<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<p>See the following topics for more information:</p>
-<ul>
-<li><a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> for guidance to determine if compaction will help in your case.</li>
-<li><a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a> for how to enable and configure automatic compaction.</li>
-</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/data-management.html"><span class="arrow-prev">← </span><span>Data management</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/automatic-compaction.html"><span>Automatic compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#compaction-strategies">Compaction strategies</a></li><li [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="../data-management/compaction.html">
+<meta http-equiv="refresh" content="0; url=../data-management/compaction.html">
+<h1>Redirecting...</h1>
+<a href="../data-management/compaction.html">Click here if you are not redirected.</a>
+<script>location="../data-management/compaction.html"</script>
diff --git a/docs/24.0.0/ingestion/data-formats.html b/docs/24.0.0/ingestion/data-formats.html
index 0909792b..c8804437 100644
--- a/docs/24.0.0/ingestion/data-formats.html
+++ b/docs/24.0.0/ingestion/data-formats.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -614,7 +614,7 @@ For details, see the Schema Registry <a href="http://docs.confluent.io/current/s
<tr><th>Field</th><th>Description</th><th>Default</th></tr>
</thead>
<tbody>
-<tr><td>useFieldDiscovery</td><td>If true, interpret all root-level fields as available fields for usage by <a href="/docs/24.0.0/ingestion/ingestion-spec.html#timestampspec"><code>timestampSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#dimensionsspec"><code>dimensionsSpec</code></a>, and <a href="/docs/24.0.0/ingestion/ingestion-spec.html#metricsspec"><code>metricsSpe [...]
+<tr><td>useFieldDiscovery</td><td>If true, interpret all root-level fields as available fields for usage by <a href="/docs/24.0.0/ingestion/ingestion-spec.html#timestampspec"><code>timestampSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#dimensionsspec"><code>dimensionsSpec</code></a>, and <a href="/docs/24.0.0/ingestion/ingestion-spec.html#metricsspec"><code>metricsSpe [...]
<tr><td>fields</td><td>Specifies the fields of interest and how they are accessed. See <a href="#field-flattening-specifications">Field flattening specifications</a> for more detail.</td><td><code>[]</code></td></tr>
</tbody>
</table>
@@ -637,7 +637,7 @@ For details, see the Schema Registry <a href="http://docs.confluent.io/current/s
<tr><th>Field</th><th>Description</th><th>Default</th></tr>
</thead>
<tbody>
-<tr><td>type</td><td>Options are as follows:<br><br><ul><li><code>root</code>, referring to a field at the root level of the record. Only really useful if <code>useFieldDiscovery</code> is false.</li><li><code>path</code>, referring to a field using <a href="https://github.com/jayway/JsonPath">JsonPath</a> notation. Supported by most data formats that offer nesting, including <code>avro</code>, <code>json</code>, <code>orc</code>, and <code>parquet</code>.</li><li><code>jq</code>, referr [...]
+<tr><td>type</td><td>Options are as follows:<br /><br /><ul><li><code>root</code>, referring to a field at the root level of the record. Only really useful if <code>useFieldDiscovery</code> is false.</li><li><code>path</code>, referring to a field using <a href="https://github.com/jayway/JsonPath">JsonPath</a> notation. Supported by most data formats that offer nesting, including <code>avro</code>, <code>json</code>, <code>orc</code>, and <code>parquet</code>.</li><li><code>jq</code>, re [...]
<tr><td>name</td><td>Name of the field after flattening. This name can be referred to by the <a href="/docs/24.0.0/ingestion/ingestion-spec.html#timestampspec"><code>timestampSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a>, <a href="/docs/24.0.0/ingestion/ingestion-spec.html#dimensionsspec"><code>dimensionsSpec</code></a>, and <a href="/docs/24.0.0/ingestion/ingestion-spec.html#metricsspec"><code>metricsSpec</code></a>.</ [...]
<tr><td>expr</td><td>Expression for accessing the field while flattening. For type <code>path</code>, this should be <a href="https://github.com/jayway/JsonPath">JsonPath</a>. For type <code>jq</code>, this should be <a href="https://github.com/eiiches/jackson-jq">jackson-jq</a> notation. For other types, this parameter is ignored.</td><td>none (required for types <code>path</code> and <code>jq</code>)</td></tr>
</tbody>
diff --git a/docs/24.0.0/ingestion/data-management.html b/docs/24.0.0/ingestion/data-management.html
index 7dd97213..82ec5ed5 100644
--- a/docs/24.0.0/ingestion/data-management.html
+++ b/docs/24.0.0/ingestion/data-management.html
@@ -1,168 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data management · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/ingestion/data-management.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/data-management.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data management</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-Within the context of this topic data management refers to Apache Druid's data maintenance capabilities for existing datasources. There are several options to help you keep your data relevant and to help your Druid cluster remain performant. For example updating, reingesting, adding lookups, reindexing, or deleting data.
-<p>In addition to the tasks covered on this page, you can also use segment compaction to improve the layout of your existing data. Refer to <a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a> to see if compaction will help in your environment. For an overview and steps to configure manual compaction tasks, see <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a>.</p>
-<h2><a class="anchor" aria-hidden="true" id="adding-new-data-to-existing-datasources"></a><a href="#adding-new-data-to-existing-datasources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 [...]
-<p>Druid can insert new data to an existing datasource by appending new segments to existing segment sets. It can also add new data by merging an existing set of segments with new data and overwriting the original set.</p>
-<p>Druid does not support single-record updates by primary key.</p>
-<p><a name="update"></a></p>
-<h2><a class="anchor" aria-hidden="true" id="updating-existing-data"></a><a href="#updating-existing-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
-<p>Once you ingest some data in a dataSource for an interval and create Apache Druid segments, you might want to make changes to
-the ingested data. There are several ways this can be done.</p>
-<h3><a class="anchor" aria-hidden="true" id="using-lookups"></a><a href="#using-lookups" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
-<p>If you have a dimension where values need to be updated frequently, try first using <a href="/docs/24.0.0/querying/lookups.html">lookups</a>. A
-classic use case of lookups is when you have an ID dimension stored in a Druid segment, and want to map the ID dimension to a
-human-readable String value that may need to be updated periodically.</p>
-<h3><a class="anchor" aria-hidden="true" id="reingesting-data"></a><a href="#reingesting-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
-<p>If lookup-based techniques are not sufficient, you will need to reingest data into Druid for the time chunks that you
-want to update. This can be done using one of the <a href="/docs/24.0.0/ingestion/index.html#batch">batch ingestion methods</a> in overwrite mode (the
-default mode). It can also be done using <a href="/docs/24.0.0/ingestion/index.html#streaming">streaming ingestion</a>, provided you drop data for the
-relevant time chunks first.</p>
-<p>If you do the reingestion in batch mode, Druid's atomic update mechanism means that queries will flip seamlessly from
-the old data to the new data.</p>
-<p>We recommend keeping a copy of your raw data around in case you ever need to reingest it.</p>
-<h3><a class="anchor" aria-hidden="true" id="with-hadoop-based-ingestion"></a><a href="#with-hadoop-based-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
-<p>This section assumes you understand how to do batch ingestion using Hadoop. See
-<a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop batch ingestion</a> for more information. Hadoop batch-ingestion can be used for reindexing and delta ingestion.</p>
-<p>Druid uses an <code>inputSpec</code> in the <code>ioConfig</code> to know where the data to be ingested is located and how to read it.
-For simple Hadoop batch ingestion, <code>static</code> or <code>granularity</code> spec types allow you to read data stored in deep storage.</p>
-<p>There are other types of <code>inputSpec</code> to enable reindexing and delta ingestion.</p>
-<h3><a class="anchor" aria-hidden="true" id="reindexing-with-native-batch-ingestion"></a><a href="#reindexing-with-native-batch-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1. [...]
-<p>This section assumes you understand how to do batch ingestion without Hadoop using <a href="/docs/24.0.0/ingestion/native-batch.html">native batch indexing</a>. Native batch indexing uses an <code>inputSource</code> to know where and how to read the input data. You can use the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>DruidInputSource</code></a> to read data from segments inside Druid. You can use Parallel task (<code>index_parallel</code>) for all native [...]
-<p><a name="delete"></a></p>
-<h2><a class="anchor" aria-hidden="true" id="deleting-data"></a><a href="#deleting-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
-<p>Druid supports permanent deletion of segments that are in an "unused" state (see the
-<a href="/docs/24.0.0/design/architecture.html#segment-lifecycle">Segment lifecycle</a> section of the Architecture page).</p>
-<p>The Kill Task deletes unused segments within a specified interval from metadata storage and deep storage.</p>
-<p>For more information, please see <a href="/docs/24.0.0/ingestion/tasks.html#kill">Kill Task</a>.</p>
-<p>Permanent deletion of a segment in Apache Druid has two steps:</p>
-<ol>
-<li>The segment must first be marked as "unused". This occurs when a segment is dropped by retention rules, and when a user manually disables a segment through the Coordinator API.</li>
-<li>After segments have been marked as "unused", a Kill Task will delete any "unused" segments from Druid's metadata store as well as deep storage.</li>
-</ol>
-<p>For documentation on retention rules, please see <a href="/docs/24.0.0/operations/rule-configuration.html">Data Retention</a>.</p>
-<p>For documentation on disabling segments using the Coordinator API, please see the
-<a href="/docs/24.0.0/operations/api-reference.html#coordinator-datasources">Coordinator Datasources API</a> reference.</p>
-<p>A data deletion tutorial is available at <a href="/docs/24.0.0/tutorials/tutorial-delete-data.html">Tutorial: Deleting data</a></p>
-<h2><a class="anchor" aria-hidden="true" id="kill-task"></a><a href="#kill-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
-<p>The kill task deletes all information about segments and removes them from deep storage. Segments to kill must be unused (used==0) in the Druid segment table.</p>
-<p>The available grammar is:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"kill"</span>,
- <span class="hljs-attr">"id"</span>: <task_id>,
- <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
- <span class="hljs-attr">"interval"</span> : <all_segments_in_this_interval_will_die!>,
- <span class="hljs-attr">"markAsUnused"</span>: <<span class="hljs-literal">true</span>|<span class="hljs-literal">false</span>>,
- <span class="hljs-attr">"context"</span>: <task context>
-}
-</code></pre>
-<p>If <code>markAsUnused</code> is true (default is false), the kill task will first mark any segments within the specified interval as unused, before deleting the unused segments within the interval.</p>
-<p><strong>WARNING!</strong> The kill task permanently removes all information about the affected segments from the metadata store and deep storage. These segments cannot be recovered after the kill task runs, this operation cannot be undone.</p>
-<h2><a class="anchor" aria-hidden="true" id="retention"></a><a href="#retention" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
-<p>Druid supports retention rules, which are used to define intervals of time where data should be preserved, and intervals where data should be discarded.</p>
-<p>Druid also supports separating Historical processes into tiers, and the retention rules can be configured to assign data for specific intervals to specific tiers.</p>
-<p>These features are useful for performance/cost management; a common use case is separating Historical processes into a "hot" tier and a "cold" tier.</p>
-<p>For more information, please see <a href="/docs/24.0.0/operations/rule-configuration.html">Load rules</a>.</p>
-<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<p>See the following topics for more information:</p>
-<ul>
-<li><a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a> for an overview and steps to configure manual compaction tasks.</li>
-<li><a href="/docs/24.0.0/design/segments.html">Segments</a> for information on how Druid handles segment versioning.</li>
-</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/schema-design.html"><span class="arrow-prev">← </span><span>Schema design tips</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/compaction.html"><span>Compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#adding-new-data-to-existing-datasources">Adding new data to existing dataso [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="../data-management/index.html">
+<meta http-equiv="refresh" content="0; url=../data-management/index.html">
+<h1>Redirecting...</h1>
+<a href="../data-management/index.html">Click here if you are not redirected.</a>
+<script>location="../data-management/index.html"</script>
diff --git a/docs/24.0.0/ingestion/data-model.html b/docs/24.0.0/ingestion/data-model.html
index f3176026..e0063e13 100644
--- a/docs/24.0.0/ingestion/data-model.html
+++ b/docs/24.0.0/ingestion/data-model.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -80,7 +80,7 @@
<p>Druid stores data in datasources, which are similar to tables in a traditional relational database management system (RDBMS). Druid's data model shares similarities with both relational and timeseries data models.</p>
<h2><a class="anchor" aria-hidden="true" id="primary-timestamp"></a><a href="#primary-timestamp" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
<p>Druid schemas must always include a primary timestamp. Druid uses the primary timestamp to <a href="/docs/24.0.0/ingestion/partitioning.html">partition and sort</a> your data. Druid uses the primary timestamp to rapidly identify and retrieve data within the time range of queries. Druid also uses the primary timestamp column
-for time-based <a href="/docs/24.0.0/ingestion/data-management.html">data management operations</a> such as dropping time chunks, overwriting time chunks, and time-based retention rules.</p>
+for time-based <a href="/docs/24.0.0/data-management/index.html">data management operations</a> such as dropping time chunks, overwriting time chunks, and time-based retention rules.</p>
<p>Druid parses the primary timestamp based on the <a href="/docs/24.0.0/ingestion/ingestion-spec.html#timestampspec"><code>timestampSpec</code></a> configuration at ingestion time. Regardless of the source field for the primary timestamp, Druid always stores the timestamp in the <code>__time</code> column in your Druid datasource.</p>
<p>You can control other important operations that are based on the primary timestamp in the
<a href="/docs/24.0.0/ingestion/ingestion-spec.html#granularityspec"><code>granularitySpec</code></a>. If you have more than one timestamp column, you can store the others as
diff --git a/docs/24.0.0/ingestion/delete-data.html b/docs/24.0.0/ingestion/delete-data.html
index 83382a66..4013695a 100644
--- a/docs/24.0.0/ingestion/delete-data.html
+++ b/docs/24.0.0/ingestion/delete-data.html
@@ -1,8 +1,8 @@
<!DOCTYPE html>
<meta charset="utf-8">
<title>Redirecting...</title>
-<link rel="canonical" href="data-management.html#delete">
-<meta http-equiv="refresh" content="0; url=data-management.html#delete">
+<link rel="canonical" href="../data-management/delete.html">
+<meta http-equiv="refresh" content="0; url=../data-management/delete.html">
<h1>Redirecting...</h1>
-<a href="data-management.html#delete">Click here if you are not redirected.</a>
-<script>location="data-management.html#delete"</script>
+<a href="../data-management/delete.html">Click here if you are not redirected.</a>
+<script>location="../data-management/delete.html"</script>
diff --git a/docs/24.0.0/ingestion/faq.html b/docs/24.0.0/ingestion/faq.html
index 751bd52e..b3ebaa49 100644
--- a/docs/24.0.0/ingestion/faq.html
+++ b/docs/24.0.0/ingestion/faq.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -79,10 +79,8 @@
-->
<h2><a class="anchor" aria-hidden="true" id="batch-ingestion"></a><a href="#batch-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
<p>If you are trying to batch load historical data but no events are being loaded, make sure the interval of your ingestion spec actually encapsulates the interval of your data. Events outside this interval are dropped.</p>
-<h2><a class="anchor" aria-hidden="true" id="druid-ingested-my-events-but-i-they-are-not-in-my-query-results"></a><a href="#druid-ingested-my-events-but-i-they-are-not-in-my-query-results" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H [...]
+<h2><a class="anchor" aria-hidden="true" id="druid-ingested-my-events-but-they-are-not-in-my-query-results"></a><a href="#druid-ingested-my-events-but-they-are-not-in-my-query-results" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-. [...]
<p>If the number of ingested events seem correct, make sure your query is correctly formed. If you included a <code>count</code> aggregator in your ingestion spec, you will need to query for the results of this aggregate with a <code>longSum</code> aggregator. Issuing a query with a count aggregator will count the number of Druid rows, which includes <a href="/docs/24.0.0/design/index.html">roll-up</a>.</p>
-<h2><a class="anchor" aria-hidden="true" id="what-types-of-data-does-druid-support"></a><a href="#what-types-of-data-does-druid-support" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 [...]
-<p>Druid can ingest JSON, CSV, TSV and other delimited data out of the box. Druid supports single dimension values, or multiple dimension values (an array of strings). Druid supports long, float, and double numeric columns.</p>
<h2><a class="anchor" aria-hidden="true" id="where-do-my-druid-segments-end-up-after-ingestion"></a><a href="#where-do-my-druid-segments-end-up-after-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 [...]
<p>Depending on what <code>druid.storage.type</code> is set to, Druid will upload segments to some <a href="/docs/24.0.0/dependencies/deep-storage.html">Deep Storage</a>. Local disk is used as the default deep storage.</p>
<h2><a class="anchor" aria-hidden="true" id="my-stream-ingest-is-not-handing-segments-off"></a><a href="#my-stream-ingest-is-not-handing-segments-off" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v [...]
@@ -111,28 +109,16 @@ Continue polling until all segments are loaded. Once all segments are loaded you
Note that this workflow only guarantees that the segments are available at the time of the <a href="/docs/24.0.0/operations/api-reference.html#segment-loading-by-datasource">Segment Loading by Datasource API</a> call. Segments can still become missing because of historical process failures or any other reasons afterward.</li>
</ol>
<h2><a class="anchor" aria-hidden="true" id="i-dont-see-my-druid-segments-on-my-historical-processes"></a><a href="#i-dont-see-my-druid-segments-on-my-historical-processes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22- [...]
-<p>You can check the <a href="/docs/24.0.0/operations/druid-console.html">web console</a> to make sure that your segments have actually loaded on <a href="/docs/24.0.0/design/historical.html">Historical processes</a>. If your segments are not present, check the Coordinator logs for messages about capacity of replication errors. One reason that segments are not downloaded is because Historical processes have maxSizes that are too small, making them incapable of downloading more data. You [...]
+<p>You can check the <a href="/docs/24.0.0/operations/web-console.html">web console</a> to make sure that your segments have actually loaded on <a href="/docs/24.0.0/design/historical.html">Historical processes</a>. If your segments are not present, check the Coordinator logs for messages about capacity of replication errors. One reason that segments are not downloaded is because Historical processes have maxSizes that are too small, making them incapable of downloading more data. You ca [...]
<pre><code class="hljs"><span class="hljs-attr">-Ddruid.segmentCache.locations</span>=[{<span class="hljs-string">"path"</span>:<span class="hljs-string">"/tmp/druid/storageLocation"</span>,<span class="hljs-string">"maxSize"</span>:<span class="hljs-string">"500000000000"</span>}]
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="my-queries-are-returning-empty-results"></a><a href="#my-queries-are-returning-empty-results" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1. [...]
<p>You can use a <a href="/docs/24.0.0/querying/segmentmetadataquery.html">segment metadata query</a> for the dimensions and metrics that have been created for your datasource. Make sure that the name of the aggregators you use in your query match one of these metrics. Also make sure that the query interval you specify match a valid time range where data exists.</p>
-<h2><a class="anchor" aria-hidden="true" id="how-can-i-reindex-existing-data-in-druid-with-schema-changes"></a><a href="#how-can-i-reindex-existing-data-in-druid-with-schema-changes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 [...]
-<p>You can use DruidInputSource with the <a href="/docs/24.0.0/ingestion/native-batch.html">Parallel task</a> to ingest existing druid segments using a new schema and change the name, dimensions, metrics, rollup, etc. of the segment.
-See <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">DruidInputSource</a> for more details.
-Or, if you use hadoop based ingestion, then you can use "dataSource" input spec to do reindexing.</p>
-<p>See the <a href="/docs/24.0.0/ingestion/data-management.html#update">Update existing data</a> section of the data management page for more details.</p>
-<h2><a class="anchor" aria-hidden="true" id="how-can-i-change-the-query-granularity-of-existing-data-in-druid"></a><a href="#how-can-i-change-the-query-granularity-of-existing-data-in-druid" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 [...]
-<p>In a lot of situations you may want coarser granularity for older data. Example, any data older than 1 month has only hour level granularity but newer data has minute level granularity. This use case is same as re-indexing.</p>
-<p>To do this use the <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">DruidInputSource</a> and run a <a href="/docs/24.0.0/ingestion/native-batch.html">Parallel task</a>. The DruidInputSource will allow you to take in existing segments from Druid and aggregate them and feed them back into Druid. It will also allow you to filter the data in those segments while feeding it back in. This means if there are rows you want to delete, you can just filter them away during re-ingestion.
-Typically the above will be run as a batch job to say everyday feed in a chunk of data and aggregate it.
-Or, if you use hadoop based ingestion, then you can use "dataSource" input spec to do reindexing.</p>
-<p>See the <a href="/docs/24.0.0/ingestion/data-management.html#update">Update existing data</a> section of the data management page for more details.</p>
-<p>You can also change the query granularity using compaction. See <a href="/docs/24.0.0/ingestion/compaction.html#query-granularity-handling">Query granularity handling</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="real-time-ingestion-seems-to-be-stuck"></a><a href="#real-time-ingestion-seems-to-be-stuck" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 [...]
<p>There are a few ways this can occur. Druid will throttle ingestion to prevent out of memory problems if the intermediate persists are taking too long or if hand-off is taking too long. If your process logs indicate certain columns are taking a very long time to build (for example, if your segment granularity is hourly, but creating a single column takes 30 minutes), you should re-evaluate your configuration or scale up your real-time ingestion.</p>
<h2><a class="anchor" aria-hidden="true" id="more-information"></a><a href="#more-information" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
<p>Data ingestion for Druid can be difficult for first time users. Please don't hesitate to ask questions in the <a href="https://www.druidforum.org/">Druid Forum</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/tasks.html"><span class="arrow-prev">← </span><span>Task reference</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/index.html"><span>Overview and syntax</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-ingestion">Batch Ingestion</a></li><li><a href="#druid-ingested-my-ev [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/tasks.html"><span class="arrow-prev">← </span><span>Task reference</span></a><a class="docs-next button" href="/docs/24.0.0/data-management/index.html"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#batch-ingestion">Batch Ingestion</a></li><li><a href="#druid-ingested-my-events-but-they [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/hadoop.html b/docs/24.0.0/ingestion/hadoop.html
index 87c37368..c057b543 100644
--- a/docs/24.0.0/ingestion/hadoop.html
+++ b/docs/24.0.0/ingestion/hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -577,7 +577,7 @@ Druid <a href="/docs/24.0.0/operations/other-hadoop.html">Different Hadoop Versi
</table>
<p>Please note that the command line Hadoop indexer doesn't have the locking capabilities of the indexing service, so if you choose to use it,
you have to take caution to not override segments created by real-time processing (if you that a real-time pipeline set up).</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-firehose.html"><span class="arrow-prev">← </span><span>Firehose</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/tasks.html"><span>Task reference</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#tutorial">Tutorial</a></li><li><a href="#task-syntax">Task syntax</a></li><li> [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><span class="arrow-prev">← </span><span>Native batch: input sources</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/index.html"><span>Overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#tutorial">Tutorial</a></li><li><a href="#task-syntax [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/index.html b/docs/24.0.0/ingestion/index.html
index 8a214018..56bd6a69 100644
--- a/docs/24.0.0/ingestion/index.html
+++ b/docs/24.0.0/ingestion/index.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -77,9 +77,12 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<p>Loading data in Druid is called <em>ingestion</em> or <em>indexing</em>. When you ingest data into Druid, Druid reads the data from your source system and stores it in data files called <em>segments</em>. In general, segment files contain a few million rows.</p>
-<p>For most ingestion methods, the Druid <a href="/docs/24.0.0/design/middlemanager.html">MiddleManager</a> processes or the <a href="/docs/24.0.0/design/indexer.html">Indexer</a> processes load your source data. One exception is
-Hadoop-based ingestion, which uses a Hadoop MapReduce job on YARN MiddleManager or Indexer processes to start and monitor Hadoop jobs.</p>
+<p>Loading data in Druid is called <em>ingestion</em> or <em>indexing</em>. When you ingest data into Druid, Druid reads the data from
+your source system and stores it in data files called <a href="/docs/24.0.0/design/architecture.html#datasources-and-segments"><em>segments</em></a>.
+In general, segment files contain a few million rows each.</p>
+<p>For most ingestion methods, the Druid <a href="/docs/24.0.0/design/middlemanager.html">MiddleManager</a> processes or the
+<a href="/docs/24.0.0/design/indexer.html">Indexer</a> processes load your source data. The sole exception is Hadoop-based ingestion, which
+uses a Hadoop MapReduce job on YARN.</p>
<p>During ingestion Druid creates segments and stores them in <a href="/docs/24.0.0/dependencies/deep-storage.html">deep storage</a>. Historical nodes load the segments into memory to respond to queries. For streaming ingestion, the Middle Managers and indexers can respond to queries in real-time with arriving data. See the <a href="/docs/24.0.0/design/architecture.html#storage-design">Storage design</a> section of the Druid design documentation for more information.</p>
<p>This topic introduces streaming and batch ingestion methods. The following topics describe ingestion concepts and information that apply to all <a href="#ingestion-methods">ingestion methods</a>:</p>
<ul>
@@ -95,11 +98,8 @@ the best one for your situation. Each ingestion method supports its own set of s
about how each method works, as well as configuration properties specific to that method, check out its documentation
page.</p>
<h3><a class="anchor" aria-hidden="true" id="streaming"></a><a href="#streaming" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
-<p>The most recommended, and most popular, method of streaming ingestion is the
-<a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka indexing service</a> that reads directly from Kafka. Alternatively, the Kinesis
-indexing service works with Amazon Kinesis Data Streams.</p>
-<p>Streaming ingestion uses an ongoing process called a supervisor that reads from the data stream to ingest data into Druid.</p>
-<p>This table compares the options:</p>
+<p>There are two available options for streaming ingestion. Streaming ingestion is controlled by a continuously-running
+supervisor.</p>
<table>
<thead>
<tr><th><strong>Method</strong></th><th><a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka</a></th><th><a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis</a></th></tr>
@@ -107,31 +107,29 @@ indexing service works with Amazon Kinesis Data Streams.</p>
<tbody>
<tr><td><strong>Supervisor type</strong></td><td><code>kafka</code></td><td><code>kinesis</code></td></tr>
<tr><td><strong>How it works</strong></td><td>Druid reads directly from Apache Kafka.</td><td>Druid reads directly from Amazon Kinesis.</td></tr>
-<tr><td><strong>Can ingest late data?</strong></td><td>Yes</td><td>Yes</td></tr>
-<tr><td><strong>Exactly-once guarantees?</strong></td><td>Yes</td><td>Yes</td></tr>
+<tr><td><strong>Can ingest late data?</strong></td><td>Yes.</td><td>Yes.</td></tr>
+<tr><td><strong>Exactly-once guarantees?</strong></td><td>Yes.</td><td>Yes.</td></tr>
</tbody>
</table>
<h3><a class="anchor" aria-hidden="true" id="batch"></a><a href="#batch" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09 [...]
-<p>When doing batch loads from files, you should use one-time <a href="/docs/24.0.0/ingestion/tasks.html">tasks</a>, and you have three options: <code>index_parallel</code> (native batch; parallel), <code>index_hadoop</code> (Hadoop-based),
-or <code>index</code> (native batch; single-task).</p>
-<p>In general, we recommend native batch whenever it meets your needs, since the setup is simpler (it does not depend on
-an external Hadoop cluster). However, there are still scenarios where Hadoop-based batch ingestion might be a better choice,
-for example when you already have a running Hadoop cluster and want to
-use the cluster resource of the existing cluster for batch ingestion.</p>
-<p>This table compares the three available options:</p>
+<p>There are three available options for batch ingestion. Batch ingestion jobs are associated with a controller task that
+runs for the duration of the job.</p>
<table>
<thead>
-<tr><th><strong>Method</strong></th><th><a href="/docs/24.0.0/ingestion/native-batch.html">Native batch (parallel)</a></th><th><a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop-based</a></th><th><a href="/docs/24.0.0/ingestion/native-batch-simple-task.html">Native batch (simple)</a></th></tr>
+<tr><th><strong>Method</strong></th><th><a href="/docs/24.0.0/ingestion/native-batch.html">Native batch</a></th><th><a href="/docs/24.0.0/multi-stage-query/index.html">SQL</a></th><th><a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop-based</a></th></tr>
</thead>
<tbody>
-<tr><td><strong>Task type</strong></td><td><code>index_parallel</code></td><td><code>index_hadoop</code></td><td><code>index</code></td></tr>
-<tr><td><strong>Parallel?</strong></td><td>Yes, if <code>inputFormat</code> is splittable and <code>maxNumConcurrentSubTasks</code> > 1 in <code>tuningConfig</code>. See <a href="/docs/24.0.0/ingestion/data-formats.html">data format documentation</a> for details.</td><td>Yes, always.</td><td>No. Each task is single-threaded.</td></tr>
-<tr><td><strong>Can append or overwrite?</strong></td><td>Yes, both.</td><td>Overwrite only.</td><td>Yes, both.</td></tr>
-<tr><td><strong>External dependencies</strong></td><td>None.</td><td>Hadoop cluster (Druid submits Map/Reduce jobs).</td><td>None.</td></tr>
-<tr><td><strong>Input locations</strong></td><td>Any <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>inputSource</code></a>.</td><td>Any Hadoop FileSystem or Druid datasource.</td><td>Any <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>inputSource</code></a>.</td></tr>
-<tr><td><strong>File formats</strong></td><td>Any <a href="/docs/24.0.0/ingestion/data-formats.html#input-format"><code>inputFormat</code></a>.</td><td>Any Hadoop InputFormat.</td><td>Any <a href="/docs/24.0.0/ingestion/data-formats.html#input-format"><code>inputFormat</code></a>.</td></tr>
-<tr><td><strong><a href="/docs/24.0.0/ingestion/rollup.html">Rollup modes</a></strong></td><td>Perfect if <code>forceGuaranteedRollup</code> = true in the <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig"><code>tuningConfig</code></a>.</td><td>Always perfect.</td><td>Perfect if <code>forceGuaranteedRollup</code> = true in the <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig"><code>tuningConfig</code></a>.</td></tr>
-<tr><td><strong>Partitioning options</strong></td><td>Dynamic, hash-based, and range-based partitioning methods are available. See <a href="/docs/24.0.0/ingestion/native-batch.html#partitionsspec">partitionsSpec</a> for details.</td><td>Hash-based or range-based partitioning via <a href="/docs/24.0.0/ingestion/hadoop.html#partitionsspec"><code>partitionsSpec</code></a>.</td><td>Dynamic and hash-based partitioning methods are available. See <a href="/docs/24.0.0/ingestion/native-batch.htm [...]
+<tr><td><strong>Controller task type</strong></td><td><code>index_parallel</code></td><td><code>query_controller</code></td><td><code>index_hadoop</code></td></tr>
+<tr><td><strong>How you submit it</strong></td><td>Send an <code>index_parallel</code> spec to the <a href="/docs/24.0.0/operations/api-reference.html#task-submit">task API</a>.</td><td>Send an <a href="/docs/24.0.0/multi-stage-query/concepts.html#insert">INSERT</a> or <a href="/docs/24.0.0/multi-stage-query/concepts.html#replace">REPLACE</a> statement to the <a href="/docs/24.0.0/multi-stage-query/api.html#submit-a-query">SQL task API</a>.</td><td>Send an <code>index_hadoop</code> spec [...]
+<tr><td><strong>Parallelism</strong></td><td>Using subtasks, if <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig"><code>maxNumConcurrentSubTasks</code></a> is greater than 1.</td><td>Using <code>query_worker</code> subtasks.</td><td>Using YARN.</td></tr>
+<tr><td><strong>Fault tolerance</strong></td><td>Workers automatically relaunched upon failure. Controller task failure leads to job failure.</td><td>Controller or worker task failure leads to job failure.</td><td>YARN containers automatically relaunched upon failure. Controller task failure leads to job failure.</td></tr>
+<tr><td><strong>Can append?</strong></td><td>Yes.</td><td>Yes (INSERT).</td><td>No.</td></tr>
+<tr><td><strong>Can overwrite?</strong></td><td>Yes.</td><td>Yes (REPLACE).</td><td>Yes.</td></tr>
+<tr><td><strong>External dependencies</strong></td><td>None.</td><td>None.</td><td>Hadoop cluster.</td></tr>
+<tr><td><strong>Input sources</strong></td><td>Any <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>inputSource</code></a>.</td><td>Any <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><code>inputSource</code></a> (using <a href="/docs/24.0.0/multi-stage-query/concepts.html#extern">EXTERN</a>) or Druid datasource (using FROM).</td><td>Any Hadoop FileSystem or Druid datasource.</td></tr>
+<tr><td><strong>Input formats</strong></td><td>Any <a href="/docs/24.0.0/ingestion/data-formats.html#input-format"><code>inputFormat</code></a>.</td><td>Any <a href="/docs/24.0.0/ingestion/data-formats.html#input-format"><code>inputFormat</code></a>.</td><td>Any Hadoop InputFormat.</td></tr>
+<tr><td><strong>Secondary partitioning options</strong></td><td>Dynamic, hash-based, and range-based partitioning methods are available. See <a href="/docs/24.0.0/ingestion/native-batch.html#partitionsspec">partitionsSpec</a> for details.</td><td>Range partitioning (<a href="/docs/24.0.0/multi-stage-query/concepts.html#clustering">CLUSTERED BY</a>).</td><td>Hash-based or range-based partitioning via <a href="/docs/24.0.0/ingestion/hadoop.html#partitionsspec"><code>partitionsSpec</code></ [...]
+<tr><td><strong><a href="/docs/24.0.0/ingestion/rollup.html#perfect-rollup-vs-best-effort-rollup">Rollup modes</a></strong></td><td>Perfect if <code>forceGuaranteedRollup</code> = true in the <a href="/docs/24.0.0/ingestion/native-batch.html#tuningconfig"><code>tuningConfig</code></a>.</td><td>Always perfect.</td><td>Always perfect.</td></tr>
</tbody>
</table>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/dependencies/zookeeper.html"><span class="arrow-prev">← </span><span class="function-name-prevnext">ZooKeeper</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/data-formats.html"><span>Data formats</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#ingestion-methods">Ingestion methods</a><ul class= [...]
diff --git a/docs/24.0.0/ingestion/ingestion-spec.html b/docs/24.0.0/ingestion/ingestion-spec.html
index a5aad62f..7071fec3 100644
--- a/docs/24.0.0/ingestion/ingestion-spec.html
+++ b/docs/24.0.0/ingestion/ingestion-spec.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -143,7 +143,7 @@ documentation for each <a href="/docs/24.0.0/ingestion/index.html#ingestion-meth
<p>The specific options supported by these sections will depend on the <a href="/docs/24.0.0/ingestion/index.html#ingestion-methods">ingestion method</a> you have chosen.
For more examples, refer to the documentation for each ingestion method.</p>
<p>You can also load data visually, without the need to write an ingestion spec, using the "Load data" functionality
-available in Druid's <a href="/docs/24.0.0/operations/druid-console.html">web console</a>. Druid's visual data loader supports
+available in Druid's <a href="/docs/24.0.0/operations/web-console.html">web console</a>. Druid's visual data loader supports
<a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka</a>,
<a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis</a>, and
<a href="/docs/24.0.0/ingestion/native-batch.html">native batch</a> mode.</p>
@@ -214,7 +214,7 @@ your ingestion spec.</p>
<tr><th>Field</th><th>Description</th><th>Default</th></tr>
</thead>
<tbody>
-<tr><td>column</td><td>Input row field to read the primary timestamp from.<br><br>Regardless of the name of this input field, the primary timestamp will always be stored as a column named <code>__time</code> in your Druid datasource.</td><td>timestamp</td></tr>
+<tr><td>column</td><td>Input row field to read the primary timestamp from.<br /><br />Regardless of the name of this input field, the primary timestamp will always be stored as a column named <code>__time</code> in your Druid datasource.</td><td>timestamp</td></tr>
<tr><td>format</td><td>Timestamp format. Options are: <ul><li><code>iso</code>: ISO8601 with 'T' separator, like "2000-01-01T01:02:03.456"</li><li><code>posix</code>: seconds since epoch</li><li><code>millis</code>: milliseconds since epoch</li><li><code>micro</code>: microseconds since epoch</li><li><code>nano</code>: nanoseconds since epoch</li><li><code>auto</code>: automatically detects ISO (either 'T' or space separator) or millis format</li><li>any <a href="http://joda-ti [...]
<tr><td>missingValue</td><td>Timestamp to use for input records that have a null or missing timestamp <code>column</code>. Should be in ISO8601 format, like <code>"2000-01-01T01:02:03.456"</code>, even if you have specified something else for <code>format</code>. Since Druid requires a primary timestamp, this setting can be useful for ingesting datasets that do not have any per-record timestamps at all.</td><td>none</td></tr>
</tbody>
@@ -246,8 +246,8 @@ your ingestion spec.</p>
<tr><th>Field</th><th>Description</th><th>Default</th></tr>
</thead>
<tbody>
-<tr><td><code>dimensions</code></td><td>A list of <a href="#dimension-objects">dimension names or objects</a>. You cannot include the same column in both <code>dimensions</code> and <code>dimensionExclusions</code>.<br><br>If <code>dimensions</code> and <code>spatialDimensions</code> are both null or empty arrays, Druid treats all columns other than timestamp or metrics that do not appear in <code>dimensionExclusions</code> as String-typed dimension columns. See <a href="#inclusions-and- [...]
-<tr><td><code>dimensionExclusions</code></td><td>The names of dimensions to exclude from ingestion. Only names are supported here, not objects.<br><br>This list is only used if the <code>dimensions</code> and <code>spatialDimensions</code> lists are both null or empty arrays; otherwise it is ignored. See <a href="#inclusions-and-exclusions">inclusions and exclusions</a> below for details.</td><td><code>[]</code></td></tr>
+<tr><td><code>dimensions</code></td><td>A list of <a href="#dimension-objects">dimension names or objects</a>. You cannot include the same column in both <code>dimensions</code> and <code>dimensionExclusions</code>.<br /><br />If <code>dimensions</code> and <code>spatialDimensions</code> are both null or empty arrays, Druid treats all columns other than timestamp or metrics that do not appear in <code>dimensionExclusions</code> as String-typed dimension columns. See <a href="#inclusions- [...]
+<tr><td><code>dimensionExclusions</code></td><td>The names of dimensions to exclude from ingestion. Only names are supported here, not objects.<br /><br />This list is only used if the <code>dimensions</code> and <code>spatialDimensions</code> lists are both null or empty arrays; otherwise it is ignored. See <a href="#inclusions-and-exclusions">inclusions and exclusions</a> below for details.</td><td><code>[]</code></td></tr>
<tr><td><code>spatialDimensions</code></td><td>An array of <a href="/docs/24.0.0/development/geo.html">spatial dimensions</a>.</td><td><code>[]</code></td></tr>
<tr><td><code>includeAllDimensions</code></td><td>You can set <code>includeAllDimensions</code> to true to ingest both explicit dimensions in the <code>dimensions</code> field and other dimensions that the ingestion task discovers from input data. In this case, the explicit dimensions will appear first in order that you specify them and the dimensions dynamically discovered will come after. This flag can be useful especially with auto schema discovery using <a href="./data-formats.html#f [...]
</tbody>
@@ -262,7 +262,7 @@ a <code>string</code> type dimension object with the given name, e.g. <code>&quo
</thead>
<tbody>
<tr><td>type</td><td>Either <code>string</code>, <code>long</code>, <code>float</code>, <code>double</code>, or <code>json</code>.</td><td><code>string</code></td></tr>
-<tr><td>name</td><td>The name of the dimension. This will be used as the field name to read from input records, as well as the column name stored in generated segments.<br><br>Note that you can use a <a href="#transformspec"><code>transformSpec</code></a> if you want to rename columns during ingestion time.</td><td>none (required)</td></tr>
+<tr><td>name</td><td>The name of the dimension. This will be used as the field name to read from input records, as well as the column name stored in generated segments.<br /><br />Note that you can use a <a href="#transformspec"><code>transformSpec</code></a> if you want to rename columns during ingestion time.</td><td>none (required)</td></tr>
<tr><td>createBitmapIndex</td><td>For <code>string</code> typed dimensions, whether or not bitmap indexes should be created for the column in generated segments. Creating a bitmap index requires more storage, but speeds up certain kinds of filtering (especially equality and prefix filtering). Only supported for <code>string</code> typed dimensions.</td><td><code>true</code></td></tr>
<tr><td>multiValueHandling</td><td>Specify the type of handling for <a href="/docs/24.0.0/querying/multi-value-dimensions.html">multi-value fields</a>. Possible values are <code>sorted_array</code>, <code>sorted_set</code>, and <code>array</code>. <code>sorted_array</code> and <code>sorted_set</code> order the array upon ingestion. <code>sorted_set</code> removes duplicates. <code>array</code> ingests data as-is</td><td><code>sorted_array</code></td></tr>
</tbody>
@@ -330,9 +330,9 @@ the following operations:</p>
<tbody>
<tr><td>type</td><td><code>uniform</code></td><td><code>uniform</code></td></tr>
<tr><td>segmentGranularity</td><td><a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">Time chunking</a> granularity for this datasource. Multiple segments can be created per time chunk. For example, when set to <code>day</code>, the events of the same day fall into the same time chunk which can be optionally further partitioned into multiple segments based on other configurations and input size. Any <a href="/docs/24.0.0/querying/granularities.html">granularity</a> [...]
-<tr><td>queryGranularity</td><td>The resolution of timestamp storage within each segment. This must be equal to, or finer, than <code>segmentGranularity</code>. This will be the finest granularity that you can query at and still receive sensible results, but note that you can still query at anything coarser than this granularity. E.g., a value of <code>minute</code> will mean that records will be stored at minutely granularity, and can be sensibly queried at any multiple of minutes (incl [...]
+<tr><td>queryGranularity</td><td>The resolution of timestamp storage within each segment. This must be equal to, or finer, than <code>segmentGranularity</code>. This will be the finest granularity that you can query at and still receive sensible results, but note that you can still query at anything coarser than this granularity. E.g., a value of <code>minute</code> will mean that records will be stored at minutely granularity, and can be sensibly queried at any multiple of minutes (incl [...]
<tr><td>rollup</td><td>Whether to use ingestion-time <a href="/docs/24.0.0/ingestion/rollup.html">rollup</a> or not. Note that rollup is still effective even when <code>queryGranularity</code> is set to <code>none</code>. Your data will be rolled up if they have the exactly same timestamp.</td><td><code>true</code></td></tr>
-<tr><td>intervals</td><td>A list of intervals defining time chunks for segments. Specify interval values using ISO8601 format. For example, <code>["2021-12-06T21:27:10+00:00/2021-12-07T00:00:00+00:00"]</code>. If you omit the time, the time defaults to "00:00:00".<br><br>Druid breaks the list up and rounds off the list values based on the <code>segmentGranularity</code>.<br><br>If <code>null</code> or not provided, batch ingestion tasks generally determine which time [...]
+<tr><td>intervals</td><td>A list of intervals defining time chunks for segments. Specify interval values using ISO8601 format. For example, <code>["2021-12-06T21:27:10+00:00/2021-12-07T00:00:00+00:00"]</code>. If you omit the time, the time defaults to "00:00:00".<br /><br />Druid breaks the list up and rounds off the list values based on the <code>segmentGranularity</code>.<br /><br />If <code>null</code> or not provided, batch ingestion tasks generally determine whi [...]
</tbody>
</table>
<h3><a class="anchor" aria-hidden="true" id="transformspec"></a><a href="#transformspec" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
diff --git a/docs/24.0.0/ingestion/native-batch-firehose.html b/docs/24.0.0/ingestion/native-batch-firehose.html
index 0b5bfc6a..3cc35303 100644
--- a/docs/24.0.0/ingestion/native-batch-firehose.html
+++ b/docs/24.0.0/ingestion/native-batch-firehose.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -383,7 +383,7 @@ A sample inline Firehose spec is shown below:</p>
<tr><td>delegates</td><td>List of Firehoses to combine data from</td><td>yes</td></tr>
</tbody>
</table>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><span class="arrow-prev">← </span><span>Input sources</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/hadoop.html"><span>Hadoop-based</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#statics3firehose">StaticS3Firehose</a></li><li><a href="#staticgoogle [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/select-query.html"><span class="arrow-prev">← </span><span>Select</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-simple-task.html"><span>Native batch (simple)</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#statics3firehose">StaticS3Firehose</a></li><li><a href="#staticg [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/native-batch-input-sources.html b/docs/24.0.0/ingestion/native-batch-input-sources.html
index dd7cc018..20518374 100644
--- a/docs/24.0.0/ingestion/native-batch-input-sources.html
+++ b/docs/24.0.0/ingestion/native-batch-input-sources.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -482,10 +482,8 @@ in <code>druid.ingestion.hdfs.allowedProtocols</code>. See <a href="/docs/24.0.0
<h2><a class="anchor" aria-hidden="true" id="http-input-source"></a><a href="#http-input-source" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
<p>The HTTP input source is to support reading files directly from remote sites via HTTP.</p>
<blockquote>
-<p><strong>NOTE:</strong> Ingestion tasks run under the operating system account that runs the Druid processes, for example the Indexer, Middle Manager, and Peon. This means any user who can submit an ingestion task can specify an <code>HTTPInputSource</code> at any location where the Druid process has permissions. For example, using <code>HTTPInputSource</code>, a console user has access to internal network locations where the they would be denied access otherwise.</p>
-</blockquote>
-<blockquote>
-<p><strong>WARNING:</strong> <code>HTTPInputSource</code> is not limited to the HTTP or HTTPS protocols. It uses the Java <code>URI</code> class that supports HTTP, HTTPS, FTP, file, and jar protocols by default. This means you should never run Druid under the <code>root</code> account, because a user can use the file protocol to access any files on the local disk.</p>
+<p><strong>Security notes:</strong> Ingestion tasks run under the operating system account that runs the Druid processes, for example the Indexer, Middle Manager, and Peon. This means any user who can submit an ingestion task can specify an input source referring to any location that the Druid process can access. For example, using <code>http</code> input source, users may have access to internal network servers.</p>
+<p>The <code>http</code> input source is not limited to the HTTP or HTTPS protocols. It uses the Java URI class that supports HTTP, HTTPS, FTP, file, and jar protocols by default.</p>
</blockquote>
<p>For more information about security best practices, see <a href="/docs/24.0.0/operations/security-overview.html#best-practices">Security overview</a>.</p>
<p>The HTTP input source is <em>splittable</em> and can be used by the <a href="/docs/24.0.0/ingestion/native-batch.html">Parallel task</a>,
@@ -645,9 +643,9 @@ of milliseconds since the epoch (January 1, 1970 00:00:00 UTC). It is common to
want the output timestamp to be equivalent to the input timestamp. In this case, set the timestamp column to <code>__time</code>
and the format to <code>auto</code> or <code>millis</code>.</p>
<p>It is OK for the input and output datasources to be the same. In this case, newly generated data will overwrite the
-previous data for the intervals specified in the <code>granularitySpec</code>. Generally, if you are going to do this, it is a
-good idea to test out your reindexing by writing to a separate datasource before overwriting your main one.
-Alternatively, if your goals can be satisfied by <a href="/docs/24.0.0/ingestion/compaction.html">compaction</a>, consider that instead as a simpler
+previous data for the intervals specified in the <code>granularitySpec</code>. Generally, if you are going to do this, it is a good
+idea to test out your reindexing by writing to a separate datasource before overwriting your main one. Alternatively, if
+your goals can be satisfied by <a href="/docs/24.0.0/data-management/compaction.html">compaction</a>, consider that instead as a simpler
approach.</p>
<p>An example task spec is shown below. It reads from a hypothetical raw datasource <code>wikipedia_raw</code> and creates a new
rolled-up datasource <code>wikipedia_rollup</code> by grouping on hour, "countryName", and "page".</p>
@@ -800,7 +798,7 @@ Set this value in <code>maxNumConcurrentSubTasks</code> in <code>tuningConfig</c
<li><code>hashed</code> or <code>dynamic</code> partitioning: greater than or equal to 2</li>
</ul>
<p>For more information on the <code>maxNumConcurrentSubTasks</code> field, see <a href="/docs/24.0.0/ingestion/native-batch.html#implementation-considerations">Implementation considerations</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-simple-task.html"><span class="arrow-prev">← </span><span>Simple task indexing</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-firehose.html"><span>Firehose</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#s3-input-source">S3 input source</a></li><li><a href= [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch.html"><span class="arrow-prev">← </span><span>Native batch</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/hadoop.html"><span>Hadoop-based</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#s3-input-source">S3 input source</a></li><li><a href="#google-cloud-storage-input-so [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/native-batch-simple-task.html b/docs/24.0.0/ingestion/native-batch-simple-task.html
index d5948b0d..597253e0 100644
--- a/docs/24.0.0/ingestion/native-batch-simple-task.html
+++ b/docs/24.0.0/ingestion/native-batch-simple-task.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -77,7 +77,11 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<p>The simple task (type <code>index</code>) is designed to ingest small data sets into Apache Druid. The task executes within the indexing service. For general information on native batch indexing and parallel task indexing, see <a href="/docs/24.0.0/ingestion/native-batch.html">Native batch ingestion</a>.</p>
+<blockquote>
+<p>This page describes native batch ingestion using <a href="/docs/24.0.0/ingestion/ingestion-spec.html">ingestion specs</a>. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion
+methods</a> table to determine which ingestion method is right for you.</p>
+</blockquote>
+<p>The simple task (<a href="/docs/24.0.0/ingestion/tasks.html">task type</a> <code>index</code>) executes single-threaded as a single task within the indexing service. For parallel, scalable options consider using <a href="/docs/24.0.0/ingestion/native-batch.html"><code>index_parallel</code> tasks</a> or <a href="/docs/24.0.0/multi-stage-query/index.html">SQL-based batch ingestion</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="simple-task-example"></a><a href="#simple-task-example" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
<p>A sample task is shown below:</p>
<pre><code class="hljs css language-json">{
@@ -195,7 +199,7 @@ that range if there's some stray data with unexpected timestamps.</p>
<tr><td>forceGuaranteedRollup</td><td>Forces guaranteeing the <a href="/docs/24.0.0/ingestion/rollup.html">perfect rollup</a>. The perfect rollup optimizes the total size of generated segments and querying time while indexing time will be increased. If this is set to true, the index task will read the entire input data twice: one for finding the optimal number of partitions per time chunk and one for generating segments. Note that the result segments would be hash-partitioned. This flag [...]
<tr><td>reportParseExceptions</td><td>DEPRECATED. If true, exceptions encountered during parsing will be thrown and will halt ingestion; if false, unparseable rows and fields will be skipped. Setting <code>reportParseExceptions</code> to true will override existing configurations for <code>maxParseExceptions</code> and <code>maxSavedParseExceptions</code>, setting <code>maxParseExceptions</code> to 0 and limiting <code>maxSavedParseExceptions</code> to no more than 1.</td><td>false</td>< [...]
<tr><td>pushTimeout</td><td>Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.</td><td>0</td><td>no</td></tr>
-<tr><td>segmentWriteOutMediumFactory</td><td>Segment write-out medium to use when creating segments. See <a href="#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>Not specified, the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used</td><td>no</td></tr>
+<tr><td>segmentWriteOutMediumFactory</td><td>Segment write-out medium to use when creating segments. See <a href="/docs/24.0.0/ingestion/native-batch.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>Not specified, the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used</td><td>no</td></tr>
<tr><td>logParseExceptions</td><td>If true, log an error message when a parsing exception occurs, containing information about the row where the error occurred.</td><td>false</td><td>no</td></tr>
<tr><td>maxParseExceptions</td><td>The maximum number of parse exceptions that can occur before the task halts ingestion and fails. Overridden if <code>reportParseExceptions</code> is set.</td><td>unlimited</td><td>no</td></tr>
<tr><td>maxSavedParseExceptions</td><td>When a parse exception occurs, Druid can keep track of the most recent parse exceptions. "maxSavedParseExceptions" limits how many exception instances will be saved. These saved exceptions will be made available after the task finishes in the <a href="/docs/24.0.0/ingestion/tasks.html#task-reports">task completion report</a>. Overridden if <code>reportParseExceptions</code> is set.</td><td>0</td><td>no</td></tr>
@@ -228,15 +232,6 @@ For perfect rollup, you should use <code>hashed</code>.</p>
<tr><td>maxTotalRows</td><td>Total number of rows in segments waiting for being pushed.</td><td>20000000</td><td>no</td></tr>
</tbody>
</table>
-<h3><a class="anchor" aria-hidden="true" id="segmentwriteoutmediumfactory"></a><a href="#segmentwriteoutmediumfactory" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
-<table>
-<thead>
-<tr><th>Field</th><th>Type</th><th>Description</th><th>Required</th></tr>
-</thead>
-<tbody>
-<tr><td>type</td><td>String</td><td>See <a href="/docs/24.0.0/configuration/index.html#segmentwriteoutmediumfactory">Additional Peon Configuration: SegmentWriteOutMediumFactory</a> for explanation and available options.</td><td>yes</td></tr>
-</tbody>
-</table>
<h2><a class="anchor" aria-hidden="true" id="segment-pushing-modes"></a><a href="#segment-pushing-modes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
<p>While ingesting data using the simple task indexing, Druid creates segments from the input data and pushes them. For segment pushing,
the simple task index supports the following segment pushing modes based upon your type of <a href="/docs/24.0.0/ingestion/rollup.html">rollup</a>:</p>
@@ -245,7 +240,7 @@ the simple task index supports the following segment pushing modes based upon yo
To enable bulk pushing mode, set <code>forceGuaranteedRollup</code> in your TuningConfig. You can not use bulk pushing with <code>appendToExisting</code> in your IOConfig.</li>
<li>Incremental pushing mode: Used for best-effort rollup. Druid pushes segments are incrementally during the course of the indexing task. The index task collects data and stores created segments in the memory and disks of the services running the task until the total number of collected rows exceeds <code>maxTotalRows</code>. At that point the index task immediately pushes all segments created up until that moment, cleans up pushed segments, and continues to ingest the remaining data.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch.html"><span class="arrow-prev">← </span><span>Native batch</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><span>Input sources</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#simple-task-example">Simple task example</a></li><li><a href="# [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-firehose.html"><span class="arrow-prev">← </span><span>Firehose (deprecated)</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/standalone-realtime.html"><span>Realtime Process</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#simple-task-example">Simple task example</a></li> [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/native-batch.html b/docs/24.0.0/ingestion/native-batch.html
index e25f2c49..16b0abc5 100644
--- a/docs/24.0.0/ingestion/native-batch.html
+++ b/docs/24.0.0/ingestion/native-batch.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -77,6 +77,10 @@
~ specific language governing permissions and limitations
~ under the License.
-->
+<blockquote>
+<p>This page describes native batch ingestion using <a href="/docs/24.0.0/ingestion/ingestion-spec.html">ingestion specs</a>. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion
+methods</a> table to determine which ingestion method is right for you.</p>
+</blockquote>
<p>Apache Druid supports the following types of native batch indexing tasks:</p>
<ul>
<li>Parallel task indexing (<code>index_parallel</code>) that can run multiple indexing tasks concurrently. Parallel task works well for production ingestion tasks.</li>
@@ -85,15 +89,15 @@
<p>This topic covers the configuration for <code>index_parallel</code> ingestion specs.</p>
<p>For related information on batch indexing, see:</p>
<ul>
-<li><a href="/docs/24.0.0/ingestion/native-batch-simple-task.html">Simple task indexing</a> for <code>index</code> task configuration.</li>
-<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">Native batch input sources</a> for a reference for <code>inputSource</code> configuration.</li>
-<li><a href="/docs/24.0.0/ingestion/index.html#batch">Hadoop-based vs. native batch comparison table</a> for a comparison of batch ingestion methods.</li>
-<li><a href="/docs/24.0.0/tutorials/tutorial-batch.html">Loading a file</a> for a tutorial on native batch ingestion.</li>
+<li><a href="/docs/24.0.0/ingestion/index.html#batch">Batch ingestion method comparison table</a> for a comparison of batch ingestion methods.</li>
+<li><a href="/docs/24.0.0/tutorials/tutorial-batch.html">Tutorial: Loading a file</a> for a tutorial on native batch ingestion.</li>
+<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">Input sources</a> for possible input sources.</li>
+<li><a href="/docs/24.0.0/ingestion/data-formats.html#input-format">Input formats</a> for possible input formats.</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="submit-an-indexing-task"></a><a href="#submit-an-indexing-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 [...]
<p>To run either kind of native batch indexing task you can:</p>
<ul>
-<li>Use the <strong>Load Data</strong> UI in the Druid console to define and submit an ingestion spec.</li>
+<li>Use the <strong>Load Data</strong> UI in the web console to define and submit an ingestion spec.</li>
<li>Define an ingestion spec in JSON based upon the <a href="#parallel-indexing-example">examples</a> and reference topics for batch indexing. Then POST the ingestion spec to the <a href="/docs/24.0.0/operations/api-reference.html#tasks">Indexer API endpoint</a>,
<code>/druid/indexer/v1/task</code>, the Overlord service. Alternatively you can use the indexing script included with Druid at <code>bin/post-index-task</code>.</li>
</ul>
@@ -135,7 +139,7 @@ the whole indexing process. The supervisor task splits the input data and create
<h4><a class="anchor" aria-hidden="true" id="fully-replacing-existing-segments-using-tombstones"></a><a href="#fully-replacing-existing-segments-using-tombstones" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 [...]
<p>You can set <code>dropExisting</code> flag in the <code>ioConfig</code> to true if you want the ingestion task to replace all existing segments that start and end within the intervals for your <code>granularitySpec</code>. This applies whether or not the new data covers all existing segments. <code>dropExisting</code> only applies when <code>appendToExisting</code> is false and the <code>granularitySpec</code> contains an <code>interval</code>. WARNING: this functionality is still in [...]
<p>The following examples demonstrate when to set the <code>dropExisting</code> property to true in the <code>ioConfig</code>:</p>
-<p>Consider an existing segment with an interval of 2020-01-01 to 2021-01-01 and <code>YEAR</code> <code>segmentGranularity</code>. You want to overwrite the whole interval of 2020-01-01 to 2021-01-01 with new data using the finer segmentGranularity of <code>MONTH</code>. If the replacement data does not have a record within every months from 2020-01-01 to 2021-01-01 Druid cannot drop the original <code>YEAR</code> segment even if it does include all the replacement data. Set <code>dropE [...]
+<p>Consider an existing segment with an interval of 2020-01-01 to 2021-01-01 and <code>YEAR</code> <code>segmentGranularity</code>. You want to overwrite the whole interval of 2020-01-01 to 2021-01-01 with new data using the finer segmentGranularity of <code>MONTH</code>. If the replacement data does not have a record within every months from 2020-01-01 to 2021-01-01 Druid cannot drop the original <code>YEAR</code> segment even if it does include all the replacement data. Set <code>dropE [...]
Imagine you want to re-ingest or overwrite a datasource and the new data does not contain some time intervals that exist in the datasource. For example, a datasource contains the following data at <code>MONTH</code> segmentGranularity:</p>
<ul>
<li><strong>January</strong>: 1 record</li>
@@ -286,7 +290,7 @@ Imagine you want to re-ingest or overwrite a datasource and the new data does no
<tr><td>forceGuaranteedRollup</td><td>Forces <a href="/docs/24.0.0/ingestion/rollup.html">perfect rollup</a>. The perfect rollup optimizes the total size of generated segments and querying time but increases indexing time. If true, specify <code>intervals</code> in the <code>granularitySpec</code> and use either <code>hashed</code> or <code>single_dim</code> for the <code>partitionsSpec</code>. You cannot use this flag in conjunction with <code>appendToExisting</code> of IOConfig. For mo [...]
<tr><td>reportParseExceptions</td><td>If true, Druid throws exceptions encountered during parsing and halts ingestion. If false, Druid skips unparseable rows and fields.</td><td>false</td><td>no</td></tr>
<tr><td>pushTimeout</td><td>Milliseconds to wait to push segments. Must be >= 0, where 0 means to wait forever.</td><td>0</td><td>no</td></tr>
-<tr><td>segmentWriteOutMediumFactory</td><td>Segment write-out medium to use when creating segments. See <a href="/docs/24.0.0/ingestion/native-batch-simple-task.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>If not specified, uses the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code></td><td>no</td></tr>
+<tr><td>segmentWriteOutMediumFactory</td><td>Segment write-out medium to use when creating segments. See <a href="#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>If not specified, uses the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code></td><td>no</td></tr>
<tr><td>maxNumConcurrentSubTasks</td><td>Maximum number of worker tasks that can be run in parallel at the same time. The supervisor task spawns worker tasks up to <code>maxNumConcurrentSubTasks</code> regardless of the current available task slots. If this value is 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, the supervisor may create too many worker tasks that block other ingestion tasks. See <a href="#c [...]
<tr><td>maxRetry</td><td>Maximum number of retries on task failures.</td><td>3</td><td>no</td></tr>
<tr><td>maxNumSegmentsToMerge</td><td>Max limit for the number of segments that a single task can merge at the same time in the second phase. Used only when <code>forceGuaranteedRollup</code> is true.</td><td>100</td><td>no</td></tr>
@@ -761,13 +765,22 @@ by assigning more task slots to them.</p>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#azure-input-source">Azure input source</a> (<code>azure</code>) reads data from Azure Blob Storage and Azure Data Lake.</li>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#hdfs-input-source">HDFS input source</a> (<code>hdfs</code>) reads data from HDFS storage.</li>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#http-input-source">HTTP input Source</a> (<code>http</code>) reads data from HTTP servers.</li>
-<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#inline-input-source">Inline input Source</a> reads data you paste into the Druid console.</li>
+<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#inline-input-source">Inline input Source</a> reads data you paste into the web console.</li>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#local-input-source">Local input Source</a> (<code>local</code>) reads data from local storage.</li>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#druid-input-source">Druid input Source</a> (<code>druid</code>) reads data from a Druid datasource.</li>
<li><a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#sql-input-source">SQL input Source</a> (<code>sql</code>) reads data from a RDBMS source.</li>
</ul>
<p>For information on how to combine input sources, see <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html#combining-input-source">Combining input source</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/standalone-realtime.html"><span class="arrow-prev">← </span><span>Realtime Process</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-simple-task.html"><span>Simple task indexing</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-an-indexing-task">Submit an indexing tas [...]
+<h3><a class="anchor" aria-hidden="true" id="segmentwriteoutmediumfactory"></a><a href="#segmentwriteoutmediumfactory" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
+<table>
+<thead>
+<tr><th>Field</th><th>Type</th><th>Description</th><th>Required</th></tr>
+</thead>
+<tbody>
+<tr><td>type</td><td>String</td><td>See <a href="/docs/24.0.0/configuration/index.html#segmentwriteoutmediumfactory">Additional Peon Configuration: SegmentWriteOutMediumFactory</a> for explanation and available options.</td><td>yes</td></tr>
+</tbody>
+</table>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/standalone-realtime.html"><span class="arrow-prev">← </span><span>Realtime Process</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-input-sources.html"><span>Native batch: input sources</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-an-indexing-task">Submit an ind [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/partitioning.html b/docs/24.0.0/ingestion/partitioning.html
index 52cce5f5..4fad6f3a 100644
--- a/docs/24.0.0/ingestion/partitioning.html
+++ b/docs/24.0.0/ingestion/partitioning.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -82,18 +82,25 @@
<p>This topic describes how to set up partitions within a single datasource. It does not cover how to use multiple datasources. See <a href="/docs/24.0.0/querying/multitenancy.html">Multitenancy considerations</a> for more details on splitting data into separate datasources and potential operational considerations.</p>
<h2><a class="anchor" aria-hidden="true" id="time-chunk-partitioning"></a><a href="#time-chunk-partitioning" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 [...]
<p>Druid always partitions datasources by time into <em>time chunks</em>. Each time chunk contains one or more segments. This partitioning happens for all ingestion methods based on the <code>segmentGranularity</code> parameter in your ingestion spec <code>dataSchema</code> object.</p>
+<p>Partitioning by time is important for two reasons:</p>
+<ol>
+<li>Queries that filter by <code>__time</code> (SQL) or <code>intervals</code> (native) are able to use time partitioning to prune the set of segments to consider.</li>
+<li>Certain data management operations, such as overwriting and compacting existing data, acquire exclusive write locks on time partitions.</li>
+<li>Each segment file is wholly contained within a time partition. Too-fine-grained partitioning may cause a large number
+of small segments, which leads to poor performance.</li>
+</ol>
+<p>The most common choices to balance these considerations are <code>hour</code> and <code>day</code>. For streaming ingestion, <code>hour</code> is especially
+common, because it allows compaction to follow ingestion with less of a time delay.</p>
<h2><a class="anchor" aria-hidden="true" id="secondary-partitioning"></a><a href="#secondary-partitioning" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
<p>Druid can partition segments within a particular time chunk further depending upon options that vary based on the ingestion type you have chosen. In general, secondary partitioning on a particular dimension improves locality. This means that rows with the same value for that dimension are stored together, decreasing access time.</p>
<p>To achieve the best performance and smallest overall footprint, partition your data on a "natural"
dimension that you often use as a filter when possible. Such partitioning often improves compression and query performance. For example, some cases have yielded threefold storage size decreases.</p>
<h2><a class="anchor" aria-hidden="true" id="partitioning-and-sorting"></a><a href="#partitioning-and-sorting" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-. [...]
<p>Partitioning and sorting work well together. If you do have a "natural" partitioning dimension, consider placing it first in the <code>dimensions</code> list of your <code>dimensionsSpec</code>. This way Druid sorts rows within each segment by that column. This sorting configuration frequently improves compression more than using partitioning alone.</p>
-<blockquote>
<p>Note that Druid always sorts rows within a segment by timestamp first, even before the first dimension listed in your <code>dimensionsSpec</code>. This sorting can preclude the efficacy of dimension sorting. To work around this limitation if necessary, set your <code>queryGranularity</code> equal to <code>segmentGranularity</code> in your <a href="/docs/24.0.0/ingestion/ingestion-spec.html#granularityspec"><code>granularitySpec</code></a>. Druid will set all timestamps within the segm [...]
-</blockquote>
<h2><a class="anchor" aria-hidden="true" id="how-to-configure-partitioning"></a><a href="#how-to-configure-partitioning" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 [...]
<p>Not all ingestion methods support an explicit partitioning configuration, and not all have equivalent levels of flexibility. If you are doing initial ingestion through a less-flexible method like
-Kafka), you can use <a href="/docs/24.0.0/ingestion/data-management.html#reingesting-data">reindexing</a> or <a href="/docs/24.0.0/ingestion/compaction.html">compaction</a> to repartition your data after initial ingestion. This is a powerful technique you can use to optimally partition any data older than a certain time threshold while you continuously add new data from a stream.</p>
+Kafka), you can use <a href="/docs/24.0.0/data-management/update.html#reindex">reindexing</a> or <a href="/docs/24.0.0/data-management/compaction.html">compaction</a> to repartition your data after initial ingestion. This is a powerful technique you can use to optimally partition any data older than a certain time threshold while you continuously add new data from a stream.</p>
<p>The following table shows how each ingestion method handles partitioning:</p>
<table>
<thead>
@@ -101,16 +108,17 @@ Kafka), you can use <a href="/docs/24.0.0/ingestion/data-management.html#reinges
</thead>
<tbody>
<tr><td><a href="/docs/24.0.0/ingestion/native-batch.html">Native batch</a></td><td>Configured using <a href="/docs/24.0.0/ingestion/native-batch.html#partitionsspec"><code>partitionsSpec</code></a> inside the <code>tuningConfig</code>.</td></tr>
+<tr><td><a href="/docs/24.0.0/multi-stage-query/index.html">SQL</a></td><td>Configured using <a href="/docs/24.0.0/multi-stage-query/concepts.html#partitioning"><code>PARTITIONED BY</code></a> and <a href="/docs/24.0.0/multi-stage-query/concepts.html#clustering"><code>CLUSTERED BY</code></a>.</td></tr>
<tr><td><a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop</a></td><td>Configured using <a href="/docs/24.0.0/ingestion/hadoop.html#partitionsspec"><code>partitionsSpec</code></a> inside the <code>tuningConfig</code>.</td></tr>
-<tr><td><a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka indexing service</a></td><td>Kafka topic partitioning defines how Druid partitions the datasource. You can also <a href="/docs/24.0.0/ingestion/data-management.html#reingesting-data">reindex</a> or <a href="/docs/24.0.0/ingestion/compaction.html">compact</a> to repartition after initial ingestion.</td></tr>
-<tr><td><a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis indexing service</a></td><td>Kinesis stream sharding defines how Druid partitions the datasource. You can also <a href="/docs/24.0.0/ingestion/data-management.html#reingesting-data">reindex</a> or <a href="/docs/24.0.0/ingestion/compaction.html">compact</a> to repartition after initial ingestion.</td></tr>
+<tr><td><a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka indexing service</a></td><td>Kafka topic partitioning defines how Druid partitions the datasource. You can also <a href="/docs/24.0.0/data-management/update.html#reindex">reindex</a> or <a href="/docs/24.0.0/data-management/compaction.html">compact</a> to repartition after initial ingestion.</td></tr>
+<tr><td><a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis indexing service</a></td><td>Kinesis stream sharding defines how Druid partitions the datasource. You can also <a href="/docs/24.0.0/data-management/update.html#reindex">reindex</a> or <a href="/docs/24.0.0/data-management/compaction.html">compact</a> to repartition after initial ingestion.</td></tr>
</tbody>
</table>
<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<p>See the following topics for more information:</p>
<ul>
<li><a href="/docs/24.0.0/ingestion/native-batch.html#partitionsspec"><code>partitionsSpec</code></a> for more detail on partitioning with Native Batch ingestion.</li>
-<li><a href="/docs/24.0.0/ingestion/data-management.html#reingesting-data">Reindexing</a> and <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a> for information on how to repartition existing data in Druid.</li>
+<li><a href="/docs/24.0.0/data-management/update.html#reindex">Reindexing</a> and <a href="/docs/24.0.0/data-management/compaction.html">Compaction</a> for information on how to repartition existing data in Druid.</li>
</ul>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/rollup.html"><span class="arrow-prev">← </span><span>Data rollup</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/ingestion-spec.html"><span>Ingestion spec</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#time-chunk-partitioning">Time chunk partitioning</a></li><li><a href="#secondary- [...]
document.addEventListener('keyup', function(e) {
diff --git a/docs/24.0.0/ingestion/rollup.html b/docs/24.0.0/ingestion/rollup.html
index 41c543ac..538baead 100644
--- a/docs/24.0.0/ingestion/rollup.html
+++ b/docs/24.0.0/ingestion/rollup.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -110,7 +110,7 @@ When queries only involve dimensions in the "abbreviated" set, use the
<li>If you use a <a href="#perfect-rollup-vs-best-effort-rollup">best-effort rollup</a> ingestion configuration that does not guarantee perfect rollup, try one of the following:
<ul>
<li>Switch to a guaranteed perfect rollup option.</li>
-<li><a href="/docs/24.0.0/ingestion/data-management.html#reingesting-data">Reindex</a> or <a href="/docs/24.0.0/ingestion/compaction.html">compact</a> your data in the background after initial ingestion.</li>
+<li><a href="/docs/24.0.0/data-management/update.html#reindex">Reindex</a> or <a href="/docs/24.0.0/data-management/compaction.html">compact</a> your data in the background after initial ingestion.</li>
</ul></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="perfect-rollup-vs-best-effort-rollup"></a><a href="#perfect-rollup-vs-best-effort-rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 [...]
@@ -133,6 +133,7 @@ In both of these cases, records that could theoretically be rolled up may end up
</thead>
<tbody>
<tr><td><a href="/docs/24.0.0/ingestion/native-batch.html">Native batch</a></td><td><code>index_parallel</code> and <code>index</code> type may be either perfect or best-effort, based on configuration.</td></tr>
+<tr><td><a href="/docs/24.0.0/multi-stage-query/index.html">SQL-based batch</a></td><td>Always perfect.</td></tr>
<tr><td><a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop</a></td><td>Always perfect.</td></tr>
<tr><td><a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka indexing service</a></td><td>Always best-effort.</td></tr>
<tr><td><a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis indexing service</a></td><td>Always best-effort.</td></tr>
diff --git a/docs/24.0.0/ingestion/schema-changes.html b/docs/24.0.0/ingestion/schema-changes.html
new file mode 100644
index 00000000..a9405df2
--- /dev/null
+++ b/docs/24.0.0/ingestion/schema-changes.html
@@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="../design/segments.html#segments-with-different-schemas">
+<meta http-equiv="refresh" content="0; url=../design/segments.html#segments-with-different-schemas">
+<h1>Redirecting...</h1>
+<a href="../design/segments.html#segments-with-different-schemas">Click here if you are not redirected.</a>
+<script>location="../design/segments.html#segments-with-different-schemas"</script>
diff --git a/docs/24.0.0/ingestion/schema-design.html b/docs/24.0.0/ingestion/schema-design.html
index 10997100..9433f47a 100644
--- a/docs/24.0.0/ingestion/schema-design.html
+++ b/docs/24.0.0/ingestion/schema-design.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -263,7 +263,7 @@ some work at ETL time.</p>
<pre><code class="hljs">{ <span class="hljs-attr">"type"</span> : <span class="hljs-string">"hyperUnique"</span>, <span class="hljs-attr">"name"</span> : <span class="hljs-string">"devices"</span>, <span class="hljs-attr">"fieldName"</span> : <span class="hljs-string">"device_id_met"</span> }
</code></pre>
<p><code>device_id_dim</code> should automatically get picked up as a dimension.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/ingestion-spec.html"><span class="arrow-prev">← </span><span>Ingestion spec</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/data-management.html"><span>Data management</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#druids-data-model">Druid's data model</a></li><li><a href="#if-youre [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/ingestion-spec.html"><span class="arrow-prev">← </span><span>Ingestion spec</span></a><a class="docs-next button" href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html"><span>Apache Kafka ingestion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#druids-data-model">Druid's data model</a></ [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/standalone-realtime.html b/docs/24.0.0/ingestion/standalone-realtime.html
index 50bf797e..9ca3d6a3 100644
--- a/docs/24.0.0/ingestion/standalone-realtime.html
+++ b/docs/24.0.0/ingestion/standalone-realtime.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -91,7 +91,7 @@ suffered from limitations which made it not possible to achieve exactly once ing
removed at this time, since they were built to operate on the realtime nodes.</p>
<p>Please consider using the <a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka Indexing Service</a> or
<a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis Indexing Service</a> for stream pull ingestion instead.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/select-query.html"><span class="arrow-prev">← </span><span>Select</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/technology">Technology</a> · <a href="/use-cases">Use Cases</a> · <a href="/druid-powered">Powered by Druid</a> · <a href="/docs/24.0.0/"> [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/native-batch-simple-task.html"><span class="arrow-prev">← </span><span>Native batch (simple)</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/technology">Technology</a> · <a href="/use-cases">Use Cases</a> · <a href="/druid-powered">Powered by Druid</a [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/tasks.html b/docs/24.0.0/ingestion/tasks.html
index 639eb429..70fef78d 100644
--- a/docs/24.0.0/ingestion/tasks.html
+++ b/docs/24.0.0/ingestion/tasks.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -94,9 +94,9 @@ the Overlord APIs.</li>
<p><a name="reports"></a></p>
<h2><a class="anchor" aria-hidden="true" id="task-reports"></a><a href="#task-reports" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
<p>A report containing information about the number of rows ingested, and any parse exceptions that occurred is available for both completed tasks and running tasks.</p>
-<p>The reporting feature is supported by the <a href="/docs/24.0.0/ingestion/native-batch-simple-task.html">simple native batch task</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks.</p>
+<p>The reporting feature is supported by <a href="/docs/24.0.0/ingestion/native-batch.html">native batch tasks</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks.</p>
<h3><a class="anchor" aria-hidden="true" id="completion-report"></a><a href="#completion-report" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
-<p>After a task completes, a completion report can be retrieved at:</p>
+<p>After a task completes, if it supports reports, its report can be retrieved at:</p>
<pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD-HOST>:<OVERLORD-PORT><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>task<span class="hljs-regexp">/<task-id>/</span>reports
</code></pre>
<p>An example output is shown below:</p>
@@ -143,9 +143,6 @@ the Overlord APIs.</li>
<p>When a task is running, a live report containing ingestion state, unparseable events and moving average for number of events processed for 1 min, 5 min, 15 min time window can be retrieved at:</p>
<pre><code class="hljs">http:<span class="hljs-regexp">//</span><OVERLORD-HOST>:<OVERLORD-PORT><span class="hljs-regexp">/druid/i</span>ndexer<span class="hljs-regexp">/v1/</span>task<span class="hljs-regexp">/<task-id>/</span>reports
</code></pre>
-<p>and</p>
-<pre><code class="hljs">http:<span class="hljs-regexp">//</span><middlemanager-host>:<worker-port><span class="hljs-regexp">/druid/</span>worker<span class="hljs-regexp">/v1/</span>chat<span class="hljs-regexp">/<task-id>/</span>liveReports
-</code></pre>
<p>An example output is shown below:</p>
<pre><code class="hljs css language-json">{
<span class="hljs-attr">"ingestionStatsAndErrors"</span>: {
@@ -212,7 +209,7 @@ the Overlord APIs.</li>
<p>The <code>errorMsg</code> field shows a message describing the error that caused a task to fail. It will be null if the task was successful.</p>
<h2><a class="anchor" aria-hidden="true" id="live-reports"></a><a href="#live-reports" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
<h3><a class="anchor" aria-hidden="true" id="row-stats"></a><a href="#row-stats" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
-<p>The non-parallel <a href="/docs/24.0.0/ingestion/native-batch-simple-task.html">simple native batch task</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks support retrieval of row stats while the task is running.</p>
+<p>The <a href="/docs/24.0.0/ingestion/native-batch.html">native batch task</a>, the Hadoop batch task, and Kafka and Kinesis ingestion tasks support retrieval of row stats while the task is running.</p>
<p>The live report can be accessed with a GET to the following URL on a Peon running a task:</p>
<pre><code class="hljs">http:<span class="hljs-regexp">//</span><middlemanager-host>:<worker-port><span class="hljs-regexp">/druid/</span>worker<span class="hljs-regexp">/v1/</span>chat<span class="hljs-regexp">/<task-id>/</span>rowStats
</code></pre>
@@ -348,7 +345,7 @@ For example, Kafka indexing tasks of the same supervisor have the same groupId a
<h2><a class="anchor" aria-hidden="true" id="context-parameters"></a><a href="#context-parameters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
<p>The task context is used for various individual task configuration.
Specify task context configurations in the <code>context</code> field of the ingestion spec.
-When configuring <a href="/docs/24.0.0/ingestion/automatic-compaction.html">automatic compaction</a>, set the task context configurations in <code>taskContext</code> rather than in <code>context</code>.
+When configuring <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a>, set the task context configurations in <code>taskContext</code> rather than in <code>context</code>.
The settings get passed into the <code>context</code> field of the compaction tasks issued to MiddleManagers.</p>
<p>The following parameters apply to all task types.</p>
<table>
@@ -383,12 +380,8 @@ The settings get passed into the <code>context</code> field of the compaction ta
<p>Automatic log file deletion typically works based on the log file's 'modified' timestamp in the back-end store. Large clock skews between Druid processes and the long-term store might result in unintended behavior.</p>
</blockquote>
<h2><a class="anchor" aria-hidden="true" id="all-task-types"></a><a href="#all-task-types" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
-<h3><a class="anchor" aria-hidden="true" id="index"></a><a href="#index" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09 [...]
-<p>See <a href="/docs/24.0.0/ingestion/native-batch-simple-task.html">Native batch ingestion (simple task)</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="index_parallel"></a><a href="#index_parallel" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
<p>See <a href="/docs/24.0.0/ingestion/native-batch.html">Native batch ingestion (parallel task)</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="index_sub"></a><a href="#index_sub" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
-<p>Submitted automatically, on your behalf, by an <a href="#index_parallel"><code>index_parallel</code></a> task.</p>
<h3><a class="anchor" aria-hidden="true" id="index_hadoop"></a><a href="#index_hadoop" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
<p>See <a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop-based ingestion</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="index_kafka"></a><a href="#index_kafka" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
@@ -397,15 +390,13 @@ The settings get passed into the <code>context</code> field of the compaction ta
<h3><a class="anchor" aria-hidden="true" id="index_kinesis"></a><a href="#index_kinesis" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
<p>Submitted automatically, on your behalf, by a
<a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis-based ingestion supervisor</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="index_realtime"></a><a href="#index_realtime" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
-<p>Submitted automatically, on your behalf, by <a href="/docs/24.0.0/ingestion/tranquility.html">Tranquility</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="compact"></a><a href="#compact" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
<p>Compaction tasks merge all segments of the given interval. See the documentation on
-<a href="/docs/24.0.0/ingestion/compaction.html">compaction</a> for details.</p>
+<a href="/docs/24.0.0/data-management/compaction.html">compaction</a> for details.</p>
<h3><a class="anchor" aria-hidden="true" id="kill"></a><a href="#kill" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6 [...]
<p>Kill tasks delete all metadata about certain segments and removes them from deep storage.
-See the documentation on <a href="/docs/24.0.0/ingestion/data-management.html#delete">deleting data</a> for details.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/hadoop.html"><span class="arrow-prev">← </span><span>Hadoop-based</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/faq.html"><span>Troubleshooting FAQ</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#task-api">Task API</a></li><li><a href="#task-reports">Task reports</a><ul class="toc- [...]
+See the documentation on <a href="/docs/24.0.0/data-management/delete.html">deleting data</a> for details.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/known-issues.html"><span class="arrow-prev">← </span><span>Known issues</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/faq.html"><span>Troubleshooting FAQ</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#task-api">Task API</a></li><li><a href="#task-reports">Task reports</a>< [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/ingestion/update-existing-data.html b/docs/24.0.0/ingestion/update-existing-data.html
index aed02db3..1fb8e325 100644
--- a/docs/24.0.0/ingestion/update-existing-data.html
+++ b/docs/24.0.0/ingestion/update-existing-data.html
@@ -1,8 +1,8 @@
<!DOCTYPE html>
<meta charset="utf-8">
<title>Redirecting...</title>
-<link rel="canonical" href="data-management.html#update">
-<meta http-equiv="refresh" content="0; url=data-management.html#update">
+<link rel="canonical" href="../data-management/update.html">
+<meta http-equiv="refresh" content="0; url=../data-management/update.html">
<h1>Redirecting...</h1>
-<a href="data-management.html#update">Click here if you are not redirected.</a>
-<script>location="data-management.html#update"</script>
+<a href="../data-management/update.html">Click here if you are not redirected.</a>
+<script>location="../data-management/update.html"</script>
diff --git a/docs/24.0.0/misc/math-expr.html b/docs/24.0.0/misc/math-expr.html
index ad391594..cc797ac0 100644
--- a/docs/24.0.0/misc/math-expr.html
+++ b/docs/24.0.0/misc/math-expr.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/misc/papers-and-talks.html b/docs/24.0.0/misc/papers-and-talks.html
index f9b2e9eb..bb1ecc9d 100644
--- a/docs/24.0.0/misc/papers-and-talks.html
+++ b/docs/24.0.0/misc/papers-and-talks.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/multi-stage-query/api.html b/docs/24.0.0/multi-stage-query/api.html
index c2885d5b..eb96c54a 100644
--- a/docs/24.0.0/multi-stage-query/api.html
+++ b/docs/24.0.0/multi-stage-query/api.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>SQL-based ingestion APIs · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/api.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24. [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>SQL-based ingestion and multi-stage query task API · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/api.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docse [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-api.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion APIs</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/api.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion and multi-stage query task API</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,9 +78,13 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>The <strong>Query</strong> view in the Druid console provides the most stable experience for the multi-stage query task engine (MSQ task engine) and multi-stage query architecture. Use the UI if you do not need a programmatic interface.</p>
+<p>The <strong>Query</strong> view in the web console provides a friendly experience for the multi-stage query task engine (MSQ task
+engine) and multi-stage query architecture. We recommend using the web console if you do not need a programmatic
+interface.</p>
<p>When using the API for the MSQ task engine, the action you want to take determines the endpoint you use:</p>
<ul>
<li><code>/druid/v2/sql/task</code> endpoint: Submit a query for ingestion.</li>
@@ -88,19 +92,20 @@
</ul>
<h2><a class="anchor" aria-hidden="true" id="submit-a-query"></a><a href="#submit-a-query" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
<p>You submit queries to the MSQ task engine using the <code>POST /druid/v2/sql/task/</code> endpoint.</p>
-<h3><a class="anchor" aria-hidden="true" id="request"></a><a href="#request" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<h4><a class="anchor" aria-hidden="true" id="request"></a><a href="#request" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
<p>Currently, the MSQ task engine ignores the provided values of <code>resultFormat</code>, <code>header</code>,
<code>typesHeader</code>, and <code>sqlTypesHeader</code>. SQL SELECT queries write out their results into the task report (in the <code>multiStageQuery.payload.results.results</code> key) formatted as if <code>resultFormat</code> is an <code>array</code>.</p>
<p>For task queries similar to the <a href="/docs/24.0.0/multi-stage-query/examples.html">example queries</a>, you need to escape characters such as quotation marks (") if you use something like <code>curl</code>.
You don't need to escape characters if you use a method that can parse JSON seamlessly, such as Python.
The Python example in this topic escapes quotation marks although it's not required.</p>
-<p>The following example is the same query that you submit when you complete <a href="/docs/24.0.0/multi-stage-query/convert-json-spec.html">Convert a JSON ingestion spec</a> where you insert data into a table named <code>wikipedia</code>.</p>
+<p>The following example is the same query that you submit when you complete <a href="/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html">Convert a JSON ingestion
+spec</a> where you insert data into a table named <code>wikipedia</code>.</p>
<div class="tabs"><div class="nav-tabs"><div id="tab-group-1-tab-2" class="nav-link active" data-group="group_1" data-tab="tab-group-1-content-2">HTTP</div><div id="tab-group-1-tab-3" class="nav-link" data-group="group_1" data-tab="tab-group-1-content-3">curl</div><div id="tab-group-1-tab-4" class="nav-link" data-group="group_1" data-tab="tab-group-1-content-4">Python</div></div><div class="tab-content"><div id="tab-group-1-content-2" class="tab-pane active" data-group="group_1" tabindex [...]
<pre><code class="hljs css language-json">{<br /> <span class="hljs-attr">"query"</span>: <span class="hljs-string">"INSERT INTO wikipedia\nSELECT\n TIME_PARSE(\"timestamp\") AS __time,\n *\nFROM TABLE(\n EXTERN(\n '{\"type\": \"http\", \"uris\": [\"https://druid.apache.org/data/wikipedia.json.gz\"]}',\n '{\"type\": \"json\"}',\n '[{\"name\": \"added\", \"type\": \"long\"}, {\"name\": \"channel\", \"type\": \"string\"}, {\"name\": \"cityName\", \"type\": \"string\"}, {\"name [...]
</span></div></div><div id="tab-group-1-content-3" class="tab-pane" data-group="group_1" tabindex="-1"><div><span><pre><code class="hljs css language-bash"><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, and `port` with the values for your deployment.</span><br />curl --location --request POST <span class="hljs-string">'https://<username>:<password>@<your-instance>:<port>/druid/v2/sql/task/'</span> \<br /> --header <spa [...]
</span></div></div><div id="tab-group-1-content-4" class="tab-pane" data-group="group_1" tabindex="-1"><div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">import</span> json<br /><span class="hljs-keyword">import</span> requests<br /><br /><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, and `port` with the values for your deployment.</span><br />url = <span class="hljs-string">"https://<username>:<password [...]
</span></div></div></div></div>
-<h3><a class="anchor" aria-hidden="true" id="response"></a><a href="#response" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
+<h4><a class="anchor" aria-hidden="true" id="response"></a><a href="#response" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
<pre><code class="hljs css language-json">{
<span class="hljs-attr">"taskId"</span>: <span class="hljs-string">"query-f795a235-4dc7-4fef-abac-3ae3f9686b79"</span>,
<span class="hljs-attr">"state"</span>: <span class="hljs-string">"RUNNING"</span>,
@@ -118,12 +123,12 @@ The Python example in this topic escapes quotation marks although it's not requi
</table>
<h2><a class="anchor" aria-hidden="true" id="get-the-status-for-a-query-task"></a><a href="#get-the-status-for-a-query-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 [...]
<p>You can retrieve status of a query to see if it is still running, completed successfully, failed, or got canceled.</p>
-<h3><a class="anchor" aria-hidden="true" id="request-1"></a><a href="#request-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
+<h4><a class="anchor" aria-hidden="true" id="request-1"></a><a href="#request-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
<div class="tabs"><div class="nav-tabs"><div id="tab-group-5-tab-6" class="nav-link active" data-group="group_5" data-tab="tab-group-5-content-6">HTTP</div><div id="tab-group-5-tab-7" class="nav-link" data-group="group_5" data-tab="tab-group-5-content-7">curl</div><div id="tab-group-5-tab-8" class="nav-link" data-group="group_5" data-tab="tab-group-5-content-8">Python</div></div><div class="tab-content"><div id="tab-group-5-content-6" class="tab-pane active" data-group="group_5" tabindex [...]
</span></div></div><div id="tab-group-5-content-7" class="tab-pane" data-group="group_5" tabindex="-1"><div><span><pre><code class="hljs css language-bash"><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br />curl --location --request GET <span class="hljs-string">'https://<username>:<password>@<hostname>:<port>/druid/indexer/v1/task/<taskId>/status'< [...]
</span></div></div><div id="tab-group-5-content-8" class="tab-pane" data-group="group_5" tabindex="-1"><div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">import</span> requests<br /><br /><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br />url = <span class="hljs-string">"https://<username>:<password>@<hostname>:<port>/druid/ [...]
</span></div></div></div></div>
-<h3><a class="anchor" aria-hidden="true" id="response-1"></a><a href="#response-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<h4><a class="anchor" aria-hidden="true" id="response-1"></a><a href="#response-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<pre><code class="hljs">{
<span class="hljs-attr">"task"</span>: <span class="hljs-string">"query-3dc0c45d-34d7-4b15-86c9-cdb2d3ebfc4e"</span>,
<span class="hljs-attr">"status"</span>: {
@@ -155,12 +160,12 @@ The Python example in this topic escapes quotation marks although it's not requi
<li>If you encounter <code>500 Server Error</code> or <code>404 Not Found</code> errors, the task may be in the process of starting up or shutting down.</li>
</ul>
<p>For an explanation of the fields in a report, see <a href="#report-response-fields">Report response fields</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="request-2"></a><a href="#request-2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
+<h4><a class="anchor" aria-hidden="true" id="request-2"></a><a href="#request-2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
<div class="tabs"><div class="nav-tabs"><div id="tab-group-9-tab-10" class="nav-link active" data-group="group_9" data-tab="tab-group-9-content-10">HTTP</div><div id="tab-group-9-tab-11" class="nav-link" data-group="group_9" data-tab="tab-group-9-content-11">curl</div><div id="tab-group-9-tab-12" class="nav-link" data-group="group_9" data-tab="tab-group-9-content-12">Python</div></div><div class="tab-content"><div id="tab-group-9-content-10" class="tab-pane active" data-group="group_9" t [...]
</span></div></div><div id="tab-group-9-content-11" class="tab-pane" data-group="group_9" tabindex="-1"><div><span><pre><code class="hljs css language-bash"><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br />curl --location --request GET <span class="hljs-string">'https://<username>:<password>@<hostname>:<port>/druid/indexer/v1/task/<taskId>/reports [...]
</span></div></div><div id="tab-group-9-content-12" class="tab-pane" data-group="group_9" tabindex="-1"><div><span><pre><code class="hljs css language-python"><span class="hljs-keyword">import</span> requests<br /><br /><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br />url = <span class="hljs-string">"https://<username>:<password>@<hostname>:<port>/druid [...]
</span></div></div></div></div>
-<h3><a class="anchor" aria-hidden="true" id="response-2"></a><a href="#response-2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<h4><a class="anchor" aria-hidden="true" id="response-2"></a><a href="#response-2" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<p>The response shows an example report for a query.</p>
<p><details><summary>Show the response</summary></p>
<pre><code class="hljs css language-json">{
@@ -454,7 +459,8 @@ The Python example in this topic escapes quotation marks although it's not requi
}
}
</code></pre>
-<h3><a class="anchor" aria-hidden="true" id="report-response-fields"></a><a href="#report-response-fields" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
+<p></details></p>
+<p><a name="report-response-fields"></a></p>
<p>The following table describes the response fields when you retrieve a report for a MSQ task engine using the <code>/druid/indexer/v1/task/<taskId>/reports</code> endpoint:</p>
<table>
<thead>
@@ -470,8 +476,8 @@ The Python example in this topic escapes quotation marks although it's not requi
<tr><td>multiStageQuery.payload.status.errorReport.taskId</td><td>The task that reported the error, if known. May be a controller task or a worker task.</td></tr>
<tr><td>multiStageQuery.payload.status.errorReport.host</td><td>The hostname and port of the task that reported the error, if known.</td></tr>
<tr><td>multiStageQuery.payload.status.errorReport.stageNumber</td><td>The stage number that reported the error, if it happened during execution of a specific stage.</td></tr>
-<tr><td>multiStageQuery.payload.status.errorReport.error</td><td>Error object. Contains <code>errorCode</code> at a minimum, and may contain other fields as described in the <a href="/docs/24.0.0/multi-stage-query/concepts.html#error-codes">error code table</a>. Always present if there is an error.</td></tr>
-<tr><td>multiStageQuery.payload.status.errorReport.error.errorCode</td><td>One of the error codes from the <a href="/docs/24.0.0/multi-stage-query/concepts.html#error-codes">error code table</a>. Always present if there is an error.</td></tr>
+<tr><td>multiStageQuery.payload.status.errorReport.error</td><td>Error object. Contains <code>errorCode</code> at a minimum, and may contain other fields as described in the <a href="/docs/24.0.0/multi-stage-query/reference.html#error-codes">error code table</a>. Always present if there is an error.</td></tr>
+<tr><td>multiStageQuery.payload.status.errorReport.error.errorCode</td><td>One of the error codes from the <a href="/docs/24.0.0/multi-stage-query/reference.html#error-codes">error code table</a>. Always present if there is an error.</td></tr>
<tr><td>multiStageQuery.payload.status.errorReport.error.errorMessage</td><td>User-friendly error message. Not always present, even if there is an error.</td></tr>
<tr><td>multiStageQuery.payload.status.errorReport.exceptionStackTrace</td><td>Java stack trace in string form, if the error was due to a server-side exception.</td></tr>
<tr><td>multiStageQuery.payload.stages</td><td>Array of query stages.</td></tr>
@@ -491,17 +497,17 @@ The Python example in this topic escapes quotation marks although it's not requi
</tbody>
</table>
<h2><a class="anchor" aria-hidden="true" id="cancel-a-query-task"></a><a href="#cancel-a-query-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
-<h3><a class="anchor" aria-hidden="true" id="request-3"></a><a href="#request-3" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
+<h4><a class="anchor" aria-hidden="true" id="request-3"></a><a href="#request-3" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
<div class="tabs"><div class="nav-tabs"><div id="tab-group-13-tab-14" class="nav-link active" data-group="group_13" data-tab="tab-group-13-content-14">HTTP</div><div id="tab-group-13-tab-15" class="nav-link" data-group="group_13" data-tab="tab-group-13-content-15">curl</div><div id="tab-group-13-tab-16" class="nav-link" data-group="group_13" data-tab="tab-group-13-content-16">Python</div></div><div class="tab-content"><div id="tab-group-13-content-14" class="tab-pane active" data-group=" [...]
</span></div></div><div id="tab-group-13-content-15" class="tab-pane" data-group="group_13" tabindex="-1"><div><span><pre><code class="hljs css language-bash"><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br />curl --location --request POST <span class="hljs-string">'https://<username>:<password>@<your-instance>:<port>/druid/indexer/v1/task/<taskId> [...]
</span></div></div><div id="tab-group-13-content-16" class="tab-pane" data-group="group_13" tabindex="-1"><div><span><pre><code class="hljs"><span class="hljs-built_in">import</span> requests<br /><br /><span class="hljs-comment"># Make sure you replace `username`, `password`, `your-instance`, `port`, and `taskId` with the values for your deployment.</span><br /><span class="hljs-attr">url</span> = <span class="hljs-string">"https://<username>:<password>@<your-instance> [...]
</span></div></div></div></div>
-<h3><a class="anchor" aria-hidden="true" id="response-3"></a><a href="#response-3" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<h4><a class="anchor" aria-hidden="true" id="response-3"></a><a href="#response-3" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<pre><code class="hljs">{
<span class="hljs-attr">"task"</span>: <span class="hljs-string">"query-655efe33-781a-4c50-ae84-c2911b42d63c"</span>
}
</code></pre>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/examples.html"><span class="arrow-prev">← </span><span>Examples</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/security.html"><span>Security</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-a-query">Submit a query</a><ul class="toc-headings"><li><a href="#reque [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/concepts.html"><span class="arrow-prev">← </span><span>Key concepts</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/security.html"><span>Security</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-a-query">Submit a query</a></li><li><a href="#get-the-status-for-a- [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/concepts.html b/docs/24.0.0/multi-stage-query/concepts.html
index 51ff1bce..a1145efc 100644
--- a/docs/24.0.0/multi-stage-query/concepts.html
+++ b/docs/24.0.0/multi-stage-query/concepts.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-concepts.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion concepts</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/concepts.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion concepts</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,28 +78,136 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>This topic covers the main concepts and terminology of the multi-stage query architecture.</p>
-<h2><a class="anchor" aria-hidden="true" id="vocabulary"></a><a href="#vocabulary" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<p>You might see the following terms in the documentation or while you're using the multi-stage query architecture and task engine, such as when you view the report for a query:</p>
+<h2><a class="anchor" aria-hidden="true" id="sql-task-engine"></a><a href="#sql-task-engine" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
+<p>The <code>druid-multi-stage-query</code> extension adds a multi-stage query (MSQ) task engine that executes SQL SELECT,
+<a href="/docs/24.0.0/multi-stage-query/reference.html#insert">INSERT</a>, and <a href="/docs/24.0.0/multi-stage-query/reference.html#replace">REPLACE</a> statements as batch tasks in the indexing service,
+which execute on <a href="/docs/24.0.0/design/architecture.html#druid-services">Middle Managers</a>. INSERT and REPLACE tasks publish
+<a href="/docs/24.0.0/design/architecture.html#datasources-and-segments">segments</a> just like <a href="/docs/24.0.0/ingestion/index.html#batch">all other forms of batch
+ingestion</a>. Each query occupies at least two task slots while running: one controller task,
+and at least one worker task.</p>
+<p>You can execute queries using the MSQ task engine through the <strong>Query</strong> view in the <a href="/docs/24.0.0/operations/web-console.html">web
+console</a> or through the <a href="/docs/24.0.0/multi-stage-query/api.html"><code>/druid/v2/sql/task</code> API</a>.</p>
+<p>For more details on how SQL queries are executed using the MSQ task engine, see <a href="#multi-stage-query-tasks">multi-stage query
+tasks</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="sql-extensions"></a><a href="#sql-extensions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
+<p>To support ingestion, additional SQL functionality is available through the MSQ task engine.</p>
+<p><a name="extern"></a></p>
+<h3><a class="anchor" aria-hidden="true" id="read-external-data-with-extern"></a><a href="#read-external-data-with-extern" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>Query tasks can access external data through the EXTERN function, using any native batch <a href="/docs/24.0.0/ingestion/native-batch-input-sources.html">input
+source</a> and <a href="/docs/24.0.0/ingestion/data-formats.html#input-format">input format</a>.</p>
+<p>EXTERN can read multiple files in parallel across different worker tasks. However, EXTERN does not split individual
+files across multiple worker tasks. If you have a small number of very large input files, you can increase query
+parallelism by splitting up your input files.</p>
+<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#extern">EXTERN</a>.</p>
+<p><a name="insert"></a></p>
+<h3><a class="anchor" aria-hidden="true" id="load-data-with-insert"></a><a href="#load-data-with-insert" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>INSERT statements can create a new datasource or append to an existing datasource. In Druid SQL, unlike standard SQL,
+there is no syntactical difference between creating a table and appending data to a table. Druid does not include a
+<code>CREATE TABLE</code> statement.</p>
+<p>Nearly all SELECT capabilities are available for <code>INSERT ... SELECT</code> queries. Certain exceptions are listed on the <a href="/docs/24.0.0/multi-stage-query/known-issues.html#select">Known
+issues</a> page.</p>
+<p>INSERT statements acquire a shared lock to the target datasource. Multiple INSERT statements can run at the same time,
+for the same datasource, if your cluster has enough task slots.</p>
+<p>Like all other forms of <a href="/docs/24.0.0/ingestion/index.html#batch">batch ingestion</a>, each INSERT statement generates new segments and
+publishes them at the end of its run. For this reason, it is best suited to loading data in larger batches. Do not use
+INSERT statements to load data in a sequence of microbatches; for that, use <a href="/docs/24.0.0/ingestion/index.html#streaming">streaming
+ingestion</a> instead.</p>
+<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#insert">INSERT</a>.</p>
+<p><a name="replace"></a></p>
+<h3><a class="anchor" aria-hidden="true" id="overwrite-data-with-replace"></a><a href="#overwrite-data-with-replace" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
+<p>REPLACE statements can create a new datasource or overwrite data in an existing datasource. In Druid SQL, unlike
+standard SQL, there is no syntactical difference between creating a table and overwriting data in a table. Druid does
+not include a <code>CREATE TABLE</code> statement.</p>
+<p>REPLACE uses an <a href="/docs/24.0.0/multi-stage-query/reference.html#replace-specific-time-ranges">OVERWRITE clause</a> to determine which data to overwrite. You
+can overwrite an entire table, or a specific time range of a table. When you overwrite a specific time range, that time
+range must align with the granularity specified in the PARTITIONED BY clause.</p>
+<p>REPLACE statements acquire an exclusive write lock to the target time range of the target datasource. No other ingestion
+or compaction operations may proceed for that time range while the task is running. However, ingestion and compaction
+operations may proceed for other time ranges.</p>
+<p>Nearly all SELECT capabilities are available for <code>REPLACE ... SELECT</code> queries. Certain exceptions are listed on the <a href="/docs/24.0.0/multi-stage-query/known-issues.html#select">Known
+issues</a> page.</p>
+<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#replace">REPLACE</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="primary-timestamp"></a><a href="#primary-timestamp" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
+<p>Druid tables always include a primary timestamp named <code>__time</code>.</p>
+<p>It is common to set a primary timestamp by using <a href="/docs/24.0.0/querying/sql-scalar.html#date-and-time-functions">date and time
+functions</a>; for example: <code>TIME_FORMAT("timestamp", 'yyyy-MM-dd HH:mm:ss') AS __time</code>.</p>
+<p>The <code>__time</code> column is used for <a href="#partitioning-by-time">partitioning by time</a>. If you use <code>PARTITIONED BY ALL</code> or
+<code>PARTITIONED BY ALL TIME</code>, partitioning by time is disabled. In these cases, you do not need to include a <code>__time</code>
+column in your INSERT statement. However, Druid still creates a <code>__time</code> column in your Druid table and sets all
+timestamps to 1970-01-01 00:00:00.</p>
+<p>For more information, see <a href="/docs/24.0.0/ingestion/data-model.html#primary-timestamp">Primary timestamp</a>.</p>
+<p><a name="partitioning"></a></p>
+<h3><a class="anchor" aria-hidden="true" id="partitioning-by-time"></a><a href="#partitioning-by-time" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1 [...]
+<p>INSERT and REPLACE statements require the PARTITIONED BY clause, which determines how time-based partitioning is done.
+In Druid, data is split into one or more segments per time chunk, defined by the PARTITIONED BY granularity.</p>
+<p>Partitioning by time is important for three reasons:</p>
+<ol>
+<li>Queries that filter by <code>__time</code> (SQL) or <code>intervals</code> (native) are able to use time partitioning to prune the set of
+segments to consider.</li>
+<li>Certain data management operations, such as overwriting and compacting existing data, acquire exclusive write locks
+on time partitions. Finer-grained partitioning allows finer-grained exclusive write locks.</li>
+<li>Each segment file is wholly contained within a time partition. Too-fine-grained partitioning may cause a large number
+of small segments, which leads to poor performance.</li>
+</ol>
+<p><code>PARTITIONED BY HOUR</code> and <code>PARTITIONED BY DAY</code> are the most common choices to balance these considerations. <code>PARTITIONED BY ALL</code> is suitable if your dataset does not have a <a href="#primary-timestamp">primary timestamp</a>.</p>
+<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#partitioned-by">PARTITIONED BY</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="clustering"></a><a href="#clustering" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<p>Within each time chunk defined by <a href="#partitioning-by-time">time partitioning</a>, data can be further split by the optional
+<a href="/docs/24.0.0/multi-stage-query/reference.html#clustered-by">CLUSTERED BY</a> clause.</p>
+<p>For example, suppose you ingest 100 million rows per hour using <code>PARTITIONED BY HOUR</code> and <code>CLUSTERED BY hostName</code>. The
+ingestion task will generate segments of roughly 3 million rows — the default value of
+<a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters"><code>rowsPerSegment</code></a> — with lexicographic ranges of <code>hostName</code>s grouped into segments.</p>
+<p>Clustering is important for two reasons:</p>
+<ol>
+<li>Lower storage footprint due to improved locality, and therefore improved compressibility.</li>
+<li>Better query performance due to dimension-based segment pruning, which removes segments from consideration when they
+cannot possibly contain data matching a query's filter. This speeds up filters like <code>x = 'foo'</code> and <code>x IN ('foo', 'bar')</code>.</li>
+</ol>
+<p>To activate dimension-based pruning, these requirements must be met:</p>
<ul>
-<li><p><strong>Controller</strong>: An indexing service task of type <code>query_controller</code> that manages
-the execution of a query. There is one controller task per query.</p></li>
-<li><p><strong>Worker</strong>: Indexing service tasks of type <code>query_worker</code> that execute a
-query. There can be multiple worker tasks per query. Internally,
-the tasks process items in parallel using their processing pools (up to <code>druid.processing.numThreads</code> of execution parallelism
-within a worker task).</p></li>
-<li><p><strong>Stage</strong>: A stage of query execution that is parallelized across
-worker tasks. Workers exchange data with each other between stages.</p></li>
-<li><p><strong>Partition</strong>: A slice of data output by worker tasks. In INSERT or REPLACE
-queries, the partitions of the final stage become Druid segments.</p></li>
-<li><p><strong>Shuffle</strong>: Workers exchange data between themselves on a per-partition basis in a process called
-shuffling. During a shuffle, each output partition is sorted by a clustering key.</p></li>
+<li>Segments were generated by a REPLACE statement, not an INSERT statement.</li>
+<li>All CLUSTERED BY columns are single-valued string columns.</li>
</ul>
-<h2><a class="anchor" aria-hidden="true" id="how-the-msq-task-engine-works"></a><a href="#how-the-msq-task-engine-works" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 [...]
-<p>Query tasks, specifically queries for INSERT, REPLACE, and SELECT, execute using indexing service tasks. Every query occupies at least two task slots while running.</p>
-<p>When you submit a query task to the MSQ task engine, the following happens:</p>
+<p>If these requirements are <em>not</em> met, Druid still clusters data during ingestion, but will not be able to perform
+dimension-based segment pruning at query time. You can tell if dimension-based segment pruning is possible by using the
+<code>sys.segments</code> table to inspect the <code>shard_spec</code> for the segments generated by an ingestion query. If they are of type
+<code>range</code> or <code>single</code>, then dimension-based segment pruning is possible. Otherwise, it is not. The shard spec type is also
+available in the <strong>Segments</strong> view under the <strong>Partitioning</strong> column.</p>
+<p>For more information about syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#clustered-by">CLUSTERED BY</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="rollup"></a><a href="#rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
+<p><a href="/docs/24.0.0/ingestion/rollup.html">Rollup</a> is a technique that pre-aggregates data during ingestion to reduce the amount of data
+stored. Intermediate aggregations are stored in the generated segments, and further aggregation is done at query time.
+This reduces storage footprint and improves performance, often dramatically.</p>
+<p>To perform ingestion with rollup:</p>
+<ol>
+<li>Use GROUP BY. The columns in the GROUP BY clause become dimensions, and aggregation functions become metrics.</li>
+<li>Set <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters"><code>finalizeAggregations: false</code></a> in your context. This causes aggregation
+functions to write their internal state to the generated segments, instead of the finalized end result, and enables
+further aggregation at query time.</li>
+<li>Wrap all multi-value strings in <code>MV_TO_ARRAY(...)</code> and set <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters"><code>groupByEnableMultiValueUnnesting: false</code></a> in your context. This ensures that multi-value strings are left alone and
+remain lists, instead of being <a href="/docs/24.0.0/querying/sql-data-types.html#multi-value-strings">automatically unnested</a> by the
+GROUP BY operator.</li>
+</ol>
+<p>When you do all of these things, Druid understands that you intend to do an ingestion with rollup, and it writes
+rollup-related metadata into the generated segments. Other applications can then use <a href="/docs/24.0.0/querying/segmentmetadataquery.html"><code>segmentMetadata</code>
+queries</a> to retrieve rollup-related information.</p>
+<p>If you see the error "Encountered multi-value dimension <code>x</code> that cannot be processed with
+groupByEnableMultiValueUnnesting set to false", then wrap that column in <code>MV_TO_ARRAY(x) AS x</code>.</p>
+<p>The following <a href="/docs/24.0.0/querying/sql-aggregations.html">aggregation functions</a> are supported for rollup at ingestion time:
+<code>COUNT</code> (but switch to <code>SUM</code> at query time), <code>SUM</code>, <code>MIN</code>, <code>MAX</code>, <code>EARLIEST</code> (<a href="/docs/24.0.0/multi-stage-query/known-issues.html#select">string only</a>),
+<code>LATEST</code> (<a href="/docs/24.0.0/multi-stage-query/known-issues.html#select">string only</a>), <code>APPROX_COUNT_DISTINCT</code>, <code>APPROX_COUNT_DISTINCT_BUILTIN</code>,
+<code>APPROX_COUNT_DISTINCT_DS_HLL</code>, <code>APPROX_COUNT_DISTINCT_DS_THETA</code>, and <code>DS_QUANTILES_SKETCH</code> (but switch to
+<code>APPROX_QUANTILE_DS</code> at query time). Do not use <code>AVG</code>; instead, use <code>SUM</code> and <code>COUNT</code> at ingest time and compute the
+quotient at query time.</p>
+<p>For an example, see <a href="/docs/24.0.0/multi-stage-query/examples.html#insert-with-rollup">INSERT with rollup example</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="multi-stage-query-tasks"></a><a href="#multi-stage-query-tasks" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 [...]
+<h3><a class="anchor" aria-hidden="true" id="execution-flow"></a><a href="#execution-flow" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
+<p>When you execute a SQL statement using the task endpoint <a href="/docs/24.0.0/multi-stage-query/api.html#submit-a-query"><code>/druid/v2/sql/task</code></a>, the following
+happens:</p>
<ol>
<li><p>The Broker plans your SQL query into a native query, as usual.</p></li>
<li><p>The Broker wraps the native query into a task of type <code>query_controller</code>
@@ -107,100 +215,58 @@ and submits it to the indexing service.</p></li>
<li><p>The Broker returns the task ID to you and exits.</p></li>
<li><p>The controller task launches some number of worker tasks determined by
the <code>maxNumTasks</code> and <code>taskAssignment</code> <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters">context parameters</a>. You can set these settings individually for each query.</p></li>
-<li><p>The worker tasks execute the query.</p></li>
+<li><p>Worker tasks of type <code>query_worker</code> execute the query.</p></li>
<li><p>If the query is a SELECT query, the worker tasks send the results
back to the controller task, which writes them into its task report.
If the query is an INSERT or REPLACE query, the worker tasks generate and
publish new Druid segments to the provided datasource.</p></li>
</ol>
-<h2><a class="anchor" aria-hidden="true" id="parallelism"></a><a href="#parallelism" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>Parallelism affects performance.</p>
-<p>The <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters"><code>maxNumTasks</code></a> query parameter determines the maximum number of tasks (workers and one controller) your query will use. Generally, queries perform better with more workers. The lowest possible value of <code>maxNumTasks</code> is two (one worker and one controller), and the highest possible value is equal to the number of free task slots in your cluster.</p>
-<p>The <code>druid.worker.capacity</code> server property on each Middle Manager determines the maximum number
-of worker tasks that can run on each server at once. Worker tasks run single-threaded, which
-also determines the maximum number of processors on the server that can contribute towards
-multi-stage queries. Since data servers are shared between Historicals and
-Middle Managers, the default setting for <code>druid.worker.capacity</code> is lower than the number of
-processors on the server. Advanced users may consider enhancing parallelism by increasing this
-value to one less than the number of processors on the server. In most cases, this increase must
-be accompanied by an adjustment of the memory allotment of the Historical process,
-Middle-Manager-launched tasks, or both, to avoid memory overcommitment and server instability. If
-you are not comfortable tuning these memory usage parameters to avoid overcommitment, it is best
-to stick with the default <code>druid.worker.capacity</code>.</p>
-<h2><a class="anchor" aria-hidden="true" id="memory-usage"></a><a href="#memory-usage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
-<p>Increasing the amount of available memory can improve performance as follows:</p>
+<h3><a class="anchor" aria-hidden="true" id="parallelism"></a><a href="#parallelism" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>The <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters"><code>maxNumTasks</code></a> query parameter determines the maximum number of tasks your
+query will use, including the one <code>query_controller</code> task. Generally, queries perform better with more workers. The
+lowest possible value of <code>maxNumTasks</code> is two (one worker and one controller). Do not set this higher than the number of
+free slots available in your cluster; doing so will result in a <a href="/docs/24.0.0/multi-stage-query/reference.html#error-codes">TaskStartTimeout</a> error.</p>
+<p>When <a href="#extern">reading external data</a>, EXTERN can read multiple files in parallel across
+different worker tasks. However, EXTERN does not split individual files across multiple worker tasks. If you have a
+small number of very large input files, you can increase query parallelism by splitting up your input files.</p>
+<p>The <code>druid.worker.capacity</code> server property on each <a href="/docs/24.0.0/design/architecture.html#druid-services">Middle Manager</a>
+determines the maximum number of worker tasks that can run on each server at once. Worker tasks run single-threaded,
+which also determines the maximum number of processors on the server that can contribute towards multi-stage queries.</p>
+<h3><a class="anchor" aria-hidden="true" id="memory-usage"></a><a href="#memory-usage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>Increasing the amount of available memory can improve performance in certain cases:</p>
<ul>
<li>Segment generation becomes more efficient when data doesn't spill to disk as often.</li>
<li>Sorting stage output data becomes more efficient since available memory affects the
number of required sorting passes.</li>
</ul>
<p>Worker tasks use both JVM heap memory and off-heap ("direct") memory.</p>
-<p>On Peons launched by Middle Managers, the bulk of the JVM heap (75%) is split up into two bundles of equal size: one processor bundle and one worker bundle. Each one comprises 37.5% of the available JVM heap.</p>
-<p>The processor memory bundle is used for query processing and segment generation. Each processor bundle must
-also provides space to buffer I/O between stages. Specifically, each downstream stage requires 1 MB of buffer space for
-each upstream worker. For example, if you have 100 workers running in stage 0, and stage 1 reads from stage 0,
-then each worker in stage 1 requires 1M * 100 = 100 MB of memory for frame buffers.</p>
-<p>The worker memory bundle is used for sorting stage output data prior to shuffle. Workers can sort
-more data than fits in memory; in this case, they will switch to using disk.</p>
-<p>Worker tasks also use off-heap ("direct") memory. Set the amount of direct
-memory available (<code>-XX:MaxDirectMemorySize</code>) to at least
-<code>(druid.processing.numThreads + 1) * druid.processing.buffer.sizeBytes</code>. Increasing the
-amount of direct memory available beyond the minimum does not speed up processing.</p>
-<p>It may be necessary to override one or more memory-related parameters if you run into one of the <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> around memory usage.</p>
-<h2><a class="anchor" aria-hidden="true" id="limits"></a><a href="#limits" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
-<p>Knowing the limits for the MSQ task engine can help you troubleshoot any <a href="#error-codes">errors</a> that you encounter. Many of the errors occur as a result of reaching a limit.</p>
-<p>The following table lists query limits:</p>
-<table>
-<thead>
-<tr><th>Limit</th><th>Value</th><th>Error if exceeded</th></tr>
-</thead>
-<tbody>
-<tr><td>Size of an individual row written to a frame. Row size when written to a frame may differ from the original row size.</td><td>1 MB</td><td><code>RowTooLarge</code></td></tr>
-<tr><td>Number of segment-granular time chunks encountered during ingestion.</td><td>5,000</td><td><code>TooManyBuckets</code></td></tr>
-<tr><td>Number of input files/segments per worker.</td><td>10,000</td><td><code>TooManyInputFiles</code></td></tr>
-<tr><td>Number of output partitions for any one stage. Number of segments generated during ingestion.</td><td>25,000</td><td><code>TooManyPartitions</code></td></tr>
-<tr><td>Number of output columns for any one stage.</td><td>2,000</td><td><code>TooManyColumns</code></td></tr>
-<tr><td>Number of workers for any one stage.</td><td>Hard limit is 1,000. Memory-dependent soft limit may be lower.</td><td><code>TooManyWorkers</code></td></tr>
-<tr><td>Maximum memory occupied by broadcasted tables.</td><td>30% of each <a href="#memory-usage">processor memory bundle</a>.</td><td><code>BroadcastTablesTooLarge</code></td></tr>
-</tbody>
-</table>
-<h2><a class="anchor" aria-hidden="true" id="error-codes"></a><a href="#error-codes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>The following table describes error codes you may encounter in the <code>multiStageQuery.payload.status.errorReport.error.errorCode</code> field:</p>
-<table>
-<thead>
-<tr><th>Code</th><th>Meaning</th><th>Additional fields</th></tr>
-</thead>
-<tbody>
-<tr><td>BroadcastTablesTooLarge</td><td>The size of the broadcast tables, used in right hand side of the joins, exceeded the memory reserved for them in a worker task.</td><td><code>maxBroadcastTablesSize</code>: Memory reserved for the broadcast tables, measured in bytes.</td></tr>
-<tr><td>Canceled</td><td>The query was canceled. Common reasons for cancellation:<br /><br /><ul><li>User-initiated shutdown of the controller task via the <code>/druid/indexer/v1/task/{taskId}/shutdown</code> API.</li><li>Restart or failure of the server process that was running the controller task.</li></ul></td><td></td></tr>
-<tr><td>CannotParseExternalData</td><td>A worker task could not parse data from an external datasource.</td><td></td></tr>
-<tr><td>ColumnNameRestricted</td><td>The query uses a restricted column name.</td><td></td></tr>
-<tr><td>ColumnTypeNotSupported</td><td>Support for writing or reading from a particular column type is not supported.</td><td></td></tr>
-<tr><td>ColumnTypeNotSupported</td><td>The query attempted to use a column type that is not supported by the frame format. This occurs with ARRAY types, which are not yet implemented for frames.</td><td><code>columnName</code><br /> <br /><code>columnType</code></td></tr>
-<tr><td>InsertCannotAllocateSegment</td><td>The controller task could not allocate a new segment ID due to conflict with existing segments or pending segments. Common reasons for such conflicts:<br /> <br /><ul><li>Attempting to mix different granularities in the same intervals of the same datasource.</li><li>Prior ingestions that used non-extendable shard specs.</li></ul></td><td><code>dataSource</code><br /> <br /><code>interval</code>: The interval for the attempted new segment alloca [...]
-<tr><td>InsertCannotBeEmpty</td><td>An INSERT or REPLACE query did not generate any output rows in a situation where output rows are required for success. This can happen for INSERT or REPLACE queries with <code>PARTITIONED BY</code> set to something other than <code>ALL</code> or <code>ALL TIME</code>.</td><td><code>dataSource</code></td></tr>
-<tr><td>InsertCannotOrderByDescending</td><td>An INSERT query contained a <code>CLUSTERED BY</code> expression in descending order. Druid's segment generation code only supports ascending order.</td><td><code>columnName</code></td></tr>
-<tr><td>InsertCannotReplaceExistingSegment</td><td>A REPLACE query cannot proceed because an existing segment partially overlaps those bounds, and the portion within the bounds is not fully overshadowed by query results. <br /> <br />There are two ways to address this without modifying your query:<ul><li>Shrink the OVERLAP filter to match the query results.</li><li>Expand the OVERLAP filter to fully contain the existing segment.</li></ul></td><td><code>segmentId</code>: The existing segm [...]
-<tr><td>InsertLockPreempted</td><td>An INSERT or REPLACE query was canceled by a higher-priority ingestion job, such as a real-time ingestion task.</td><td></td></tr>
-<tr><td>InsertTimeNull</td><td>An INSERT or REPLACE query encountered a null timestamp in the <code>__time</code> field.<br /><br />This can happen due to using an expression like <code>TIME_PARSE(timestamp) AS __time</code> with a timestamp that cannot be parsed. (TIME_PARSE returns null when it cannot parse a timestamp.) In this case, try parsing your timestamps using a different function or pattern.<br /><br />If your timestamps may genuinely be null, consider using COALESCE to provid [...]
-<tr><td>InsertTimeOutOfBounds</td><td>A REPLACE query generated a timestamp outside the bounds of the TIMESTAMP parameter for your OVERWRITE WHERE clause.<br /> <br />To avoid this error, verify that the you specified is valid.</td><td><code>interval</code>: time chunk interval corresponding to the out-of-bounds timestamp</td></tr>
-<tr><td>InvalidNullByte</td><td>A string column included a null byte. Null bytes in strings are not permitted.</td><td><code>column</code>: The column that included the null byte</td></tr>
-<tr><td>QueryNotSupported</td><td>QueryKit could not translate the provided native query to a multi-stage query.<br /> <br />This can happen if the query uses features that aren't supported, like GROUPING SETS.</td><td></td></tr>
-<tr><td>RowTooLarge</td><td>The query tried to process a row that was too large to write to a single frame. See the <a href="#limits">Limits</a> table for the specific limit on frame size. Note that the effective maximum row size is smaller than the maximum frame size due to alignment considerations during frame writing.</td><td><code>maxFrameSize</code>: The limit on the frame size.</td></tr>
-<tr><td>TaskStartTimeout</td><td>Unable to launch all the worker tasks in time. <br /> <br />There might be insufficient available slots to start all the worker tasks simultaneously.<br /> <br /> Try splitting up the query into smaller chunks with lesser <code>maxNumTasks</code> number. Another option is to increase capacity.</td><td></td></tr>
-<tr><td>TooManyBuckets</td><td>Exceeded the number of partition buckets for a stage. Partition buckets are only used for <code>segmentGranularity</code> during INSERT queries. The most common reason for this error is that your <code>segmentGranularity</code> is too narrow relative to the data. See the <a href="/docs/24.0.0/multi-stage-query/concepts.html#limits">Limits</a> table for the specific limit.</td><td><code>maxBuckets</code>: The limit on buckets.</td></tr>
-<tr><td>TooManyInputFiles</td><td>Exceeded the number of input files/segments per worker. See the <a href="/docs/24.0.0/multi-stage-query/concepts.html#limits">Limits</a> table for the specific limit.</td><td><code>umInputFiles</code>: The total number of input files/segments for the stage.<br /><br /><code>maxInputFiles</code>: The maximum number of input files/segments per worker per stage.<br /><br /><code>minNumWorker</code>: The minimum number of workers required for a successful ru [...]
-<tr><td>TooManyPartitions</td><td>Exceeded the number of partitions for a stage. The most common reason for this is that the final stage of an INSERT or REPLACE query generated too many segments. See the <a href="/docs/24.0.0/multi-stage-query/concepts.html#limits">Limits</a> table for the specific limit.</td><td><code>maxPartitions</code>: The limit on partitions which was exceeded</td></tr>
-<tr><td>TooManyColumns</td><td>Exceeded the number of columns for a stage. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>maxColumns</code>: The limit on columns which was exceeded.</td></tr>
-<tr><td>TooManyWarnings</td><td>Exceeded the allowed number of warnings of a particular type.</td><td><code>rootErrorCode</code>: The error code corresponding to the exception that exceeded the required limit. <br /><br /><code>maxWarnings</code>: Maximum number of warnings that are allowed for the corresponding <code>rootErrorCode</code>.</td></tr>
-<tr><td>TooManyWorkers</td><td>Exceeded the supported number of workers running simultaneously. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>workers</code>: The number of simultaneously running workers that exceeded a hard or soft limit. This may be larger than the number of workers in any one stage if multiple stages are running simultaneously. <br /><br /><code>maxWorkers</code>: The hard or soft limit on workers that was exceeded.</td></tr>
-<tr><td>NotEnoughMemory</td><td>Insufficient memory to launch a stage.</td><td><code>serverMemory</code>: The amount of memory available to a single process.<br /><br /><code>serverWorkers</code>: The number of workers running in a single process.<br /><br /><code>serverThreads</code>: The number of threads in a single process.</td></tr>
-<tr><td>WorkerFailed</td><td>A worker task failed unexpectedly.</td><td><code>workerTaskId</code>: The ID of the worker task.</td></tr>
-<tr><td>WorkerRpcFailed</td><td>A remote procedure call to a worker task failed and could not recover.</td><td><code>workerTaskId</code>: the id of the worker task</td></tr>
-<tr><td>UnknownError</td><td>All other errors.</td><td></td></tr>
-</tbody>
-</table>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/index.html"><span class="arrow-prev">← </span><span>Overview and syntax</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/connect-external-data.html"><span>Tutorial - Load files with SQL-based ingestion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#vocabulary">Vocabul [...]
+<p>On Peons launched by Middle Managers, the bulk of the JVM heap (75%) is split up into two bundles of equal size: one
+processor bundle and one worker bundle. Each one comprises 37.5% of the available JVM heap.</p>
+<p>The processor memory bundle is used for query processing and segment generation. Each processor bundle must also
+provides space to buffer I/O between stages. Specifically, each downstream stage requires 1 MB of buffer space for each
+upstream worker. For example, if you have 100 workers running in stage 0, and stage 1 reads from stage 0, then each
+worker in stage 1 requires 1M * 100 = 100 MB of memory for frame buffers.</p>
+<p>The worker memory bundle is used for sorting stage output data prior to shuffle. Workers can sort more data than fits in
+memory; in this case, they will switch to using disk.</p>
+<p>Worker tasks also use off-heap ("direct") memory. Set the amount of direct memory available (<code>-XX:MaxDirectMemorySize</code>)
+to at least <code>(druid.processing.numThreads + 1) * druid.processing.buffer.sizeBytes</code>. Increasing the amount of direct
+memory available beyond the minimum does not speed up processing.</p>
+<h3><a class="anchor" aria-hidden="true" id="disk-usage"></a><a href="#disk-usage" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<p>Worker tasks use local disk for four purposes:</p>
+<ul>
+<li>Temporary copies of input data. Each temporary file is deleted before the next one is read. You only need
+enough temporary disk space to store one input file at a time per task.</li>
+<li>Temporary data related to segment generation. You only need enough temporary disk space to store one segments' worth
+of data at a time per task. This is generally less than 2 GB per task.</li>
+<li>External sort of data prior to shuffle. Requires enough space to store a compressed copy of the entire output dataset
+for a task.</li>
+<li>Storing stage output data during a shuffle. Requires enough space to store a compressed copy of the entire output
+dataset for a task.</li>
+</ul>
+<p>Workers use the task working directory, given by
+<a href="/docs/24.0.0/configuration/index.html#additional-peon-configuration"><code>druid.indexer.task.baseDir</code></a>, for these items. It is
+important that this directory has enough space available for these purposes.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/index.html"><span class="arrow-prev">← </span><span>Overview</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/api.html"><span>API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#sql-task-engine">SQL task engine</a></li><li><a href="#sql-extensions">SQL extensions</a><u [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/connect-external-data.html b/docs/24.0.0/multi-stage-query/connect-external-data.html
deleted file mode 100644
index 7d287b4c..00000000
--- a/docs/24.0.0/multi-stage-query/connect-external-data.html
+++ /dev/null
@@ -1,201 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial - Load files with SQL-based ingestion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/connect-external-data.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="How to generate a query that references externally hosted d [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-tutorial-connect-external-data.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Tutorial - Load files with SQL-based ingestion</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
-</blockquote>
-<p>This tutorial demonstrates how to generate a query that references externally hosted data using the <strong>Connect external data</strong> wizard.</p>
-<p>The following example uses EXTERN to query a JSON file located at <a href="https://druid.apache.org/data/wikipedia.json.gz">https://druid.apache.org/data/wikipedia.json.gz</a>.</p>
-<p>Although you can manually create a query in the UI, you can use Druid to generate a base query for you that you can modify to meet your requirements.</p>
-<p>To generate a query from external data, do the following:</p>
-<ol>
-<li><p>In the <strong>Query</strong> view of the Druid console, click <strong>Connect external data</strong>.</p></li>
-<li><p>On the <strong>Select input type</strong> screen, choose <strong>HTTP(s)</strong> and enter the following value in the <strong>URIs</strong> field: <code>https://druid.apache.org/data/wikipedia.json.gz</code>. Leave the HTTP auth username and password blank.</p></li>
-<li><p>Click <strong>Connect data</strong>.</p></li>
-<li><p>On the <strong>Parse</strong> screen, you can perform additional actions before you load the data into Druid:</p>
-<ul>
-<li>Expand a row to see what data it corresponds to from the source.</li>
-<li>Customize how Druid handles the data by selecting the <strong>Input format</strong> and its related options, such as adding <strong>JSON parser features</strong> for JSON files.</li>
-</ul></li>
-<li><p>When you're ready, click <strong>Done</strong>. You're returned to the <strong>Query</strong> view where you can see the starter query that will insert the data from the external source into a table named <code>wikipedia</code>.</p>
-<p><details><summary>Show the query</summary></p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> <span class="hljs-string">"wikipedia"</span> OVERWRITE <span class="hljs-keyword">ALL</span>
-<span class="hljs-keyword">WITH</span> ext <span class="hljs-keyword">AS</span> (<span class="hljs-keyword">SELECT</span> *
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type":"json"}'</span>,
- <span class="hljs-string">'[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"timestamp","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name": [...]
- )
-))
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- isRobot,
- channel,
- flags,
- isUnpatrolled,
- page,
- diffUrl,
- added,
- <span class="hljs-keyword">comment</span>,
- commentLength,
- isNew,
- isMinor,
- delta,
- isAnonymous,
- <span class="hljs-keyword">user</span>,
- deltaBucket,
- deleted,
- namespace,
- cityName,
- countryName,
- regionIsoCode,
- metroCode,
- countryIsoCode,
- regionName
-<span class="hljs-keyword">FROM</span> ext
-PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre>
-<p></details></p></li>
-<li><p>Review and modify the query to meet your needs. For example, you can rename the table or change segment granularity. To partition by something other than ALL, include <code>TIME_PARSE("timestamp") AS __time</code> in your SELECT statement.</p>
-<p>For example, to specify day-based segment granularity, change the partitioning to <code>PARTITIONED BY DAY</code>:</p>
-<pre><code class="hljs css language-sql"> <span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> ...
- <span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- ...
- ...
- PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre></li>
-<li><p>Optionally, select <strong>Preview</strong> to review the data before you ingest it. A preview runs the query without the REPLACE INTO clause and with an added LIMIT.
-You can see the general shape of the data before you commit to inserting it.
-The LIMITs make the query run faster but can cause incomplete results.</p></li>
-<li><p>Click <strong>Run</strong> to launch your query. The query returns information including its duration and the number of rows inserted into the table.</p></li>
-</ol>
-<h2><a class="anchor" aria-hidden="true" id="query-the-data"></a><a href="#query-the-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
-<p>You can query the <code>wikipedia</code> table after the ingestion completes.
-For example, you can analyze the data in the table to produce a list of top channels:</p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">SELECT</span>
- channel,
- <span class="hljs-keyword">COUNT</span>(*)
-<span class="hljs-keyword">FROM</span> <span class="hljs-string">"wikipedia"</span>
-<span class="hljs-keyword">GROUP</span> <span class="hljs-keyword">BY</span> channel
-<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">COUNT</span>(*) <span class="hljs-keyword">DESC</span>
-</code></pre>
-<p>With the EXTERN function, you could run the same query on the external data directly without ingesting it first:</p>
-<p><details><summary>Show the query</summary></p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">SELECT</span>
- channel,
- <span class="hljs-keyword">COUNT</span>(*)
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type": "json"}'</span>,
- <span class="hljs-string">'[{"name": "added", "type": "long"}, {"name": "channel", "type": "string"}, {"name": "cityName", "type": "string"}, {"name": "comment", "type": "string"}, {"name": "commentLength", "type": "long"}, {"name": "countryIsoCode", "type": "string"}, {"name": "countryName", "type": "string"}, {"name": "deleted", "type": "long"}, {"name": "delta", "type": "long"}, {"name": "deltaBucket", "type": "string"}, {"name": "diffUrl", "type": "string"}, {"name": "flags", "ty [...]
- )
-)
-<span class="hljs-keyword">GROUP</span> <span class="hljs-keyword">BY</span> channel
-<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">COUNT</span>(*) <span class="hljs-keyword">DESC</span>
-</code></pre>
-<p></details></p>
-<h2><a class="anchor" aria-hidden="true" id="further-reading"></a><a href="#further-reading" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
-<p>See the following topics to learn more:</p>
-<ul>
-<li><a href="/docs/24.0.0/multi-stage-query/index.html#msq-task-engine-query-syntax">MSQ task engine query syntax</a> for information about the different query components.</li>
-<li><a href="/docs/24.0.0/multi-stage-query/reference.html">Reference</a> for reference on context parameters, functions, and error codes.</li>
-</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/concepts.html"><span class="arrow-prev">← </span><span>Key concepts</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/convert-json-spec.html"><span>Tutorial - Convert an ingestion spec for SQL-based ingestion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-the-dat [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/multi-stage-query/convert-json-spec.html b/docs/24.0.0/multi-stage-query/convert-json-spec.html
deleted file mode 100644
index c67a8c98..00000000
--- a/docs/24.0.0/multi-stage-query/convert-json-spec.html
+++ /dev/null
@@ -1,233 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial - Convert an ingestion spec for SQL-based ingestion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/convert-json-spec.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="How to convert an ingestion spec to a query for S [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-tutorial-convert-ingest-spec.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Tutorial - Convert an ingestion spec for SQL-based ingestion</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
-</blockquote>
-<p>If you're already ingesting data with Druid's native SQL engine, you can use the Druid console to convert the ingestion spec to a SQL query that the multi-stage query task engine can use to ingest data.</p>
-<p>This tutorial demonstrates how to convert the ingestion spec to a query task in the Druid console.</p>
-<p>To convert the ingestion spec to a query task, do the following:</p>
-<ol>
-<li><p>In the <strong>Query</strong> view of the Druid console, navigate to the menu bar that includes <strong>Run</strong>.</p></li>
-<li><p>Click the ellipsis icon and select <strong>Convert ingestion spec to SQL</strong>.
-<img src="../assets/multi-stage-query/tutorial-msq-convert.png" alt="Convert ingestion spec to SQL" title="Convert ingestion spec to SQL"></p></li>
-<li><p>In the <strong>Ingestion spec to covert</strong> window, insert your ingestion spec. You can use your own spec or the sample ingestion spec provided in the tutorial. The sample spec uses data hosted at <code>https://druid.apache.org/data/wikipedia.json.gz</code> and loads it into a table named <code>wikipedia</code>:</p>
-<p><details><summary>Show the spec</summary></p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
- <span class="hljs-attr">"spec"</span>: {
- <span class="hljs-attr">"ioConfig"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
- <span class="hljs-attr">"inputSource"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"http"</span>,
- <span class="hljs-attr">"uris"</span>: [
- <span class="hljs-string">"https://druid.apache.org/data/wikipedia.json.gz"</span>
- ]
- },
- <span class="hljs-attr">"inputFormat"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"json"</span>
- }
- },
- <span class="hljs-attr">"tuningConfig"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
- <span class="hljs-attr">"partitionsSpec"</span>: {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"dynamic"</span>
- }
- },
- <span class="hljs-attr">"dataSchema"</span>: {
- <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
- <span class="hljs-attr">"timestampSpec"</span>: {
- <span class="hljs-attr">"column"</span>: <span class="hljs-string">"timestamp"</span>,
- <span class="hljs-attr">"format"</span>: <span class="hljs-string">"iso"</span>
- },
- <span class="hljs-attr">"dimensionsSpec"</span>: {
- <span class="hljs-attr">"dimensions"</span>: [
- <span class="hljs-string">"isRobot"</span>,
- <span class="hljs-string">"channel"</span>,
- <span class="hljs-string">"flags"</span>,
- <span class="hljs-string">"isUnpatrolled"</span>,
- <span class="hljs-string">"page"</span>,
- <span class="hljs-string">"diffUrl"</span>,
- {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
- <span class="hljs-attr">"name"</span>: <span class="hljs-string">"added"</span>
- },
- <span class="hljs-string">"comment"</span>,
- {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
- <span class="hljs-attr">"name"</span>: <span class="hljs-string">"commentLength"</span>
- },
- <span class="hljs-string">"isNew"</span>,
- <span class="hljs-string">"isMinor"</span>,
- {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
- <span class="hljs-attr">"name"</span>: <span class="hljs-string">"delta"</span>
- },
- <span class="hljs-string">"isAnonymous"</span>,
- <span class="hljs-string">"user"</span>,
- {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
- <span class="hljs-attr">"name"</span>: <span class="hljs-string">"deltaBucket"</span>
- },
- {
- <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
- <span class="hljs-attr">"name"</span>: <span class="hljs-string">"deleted"</span>
- },
- <span class="hljs-string">"namespace"</span>,
- <span class="hljs-string">"cityName"</span>,
- <span class="hljs-string">"countryName"</span>,
- <span class="hljs-string">"regionIsoCode"</span>,
- <span class="hljs-string">"metroCode"</span>,
- <span class="hljs-string">"countryIsoCode"</span>,
- <span class="hljs-string">"regionName"</span>
- ]
- },
- <span class="hljs-attr">"granularitySpec"</span>: {
- <span class="hljs-attr">"queryGranularity"</span>: <span class="hljs-string">"none"</span>,
- <span class="hljs-attr">"rollup"</span>: <span class="hljs-literal">false</span>,
- <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"day"</span>
- }
- }
- }
-}
-</code></pre>
-<p></details></p></li>
-<li><p>Click <strong>Submit</strong> to submit the spec. The Druid console uses the JSON-based ingestion spec to generate a SQL query that you can use instead. This is what the query looks like for the sample ingestion spec:</p>
-<p><details><summary>Show the query</summary></p>
-<pre><code class="hljs css language-sql"><span class="hljs-comment">-- This SQL query was auto generated from an ingestion spec</span>
-<span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> wikipedia OVERWRITE <span class="hljs-keyword">ALL</span>
-<span class="hljs-keyword">WITH</span> <span class="hljs-keyword">source</span> <span class="hljs-keyword">AS</span> (<span class="hljs-keyword">SELECT</span> * <span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type":"json"}'</span>,
- <span class="hljs-string">'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name": [...]
- )
-))
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <span class="hljs-string">"isRobot"</span>,
- <span class="hljs-string">"channel"</span>,
- <span class="hljs-string">"flags"</span>,
- <span class="hljs-string">"isUnpatrolled"</span>,
- <span class="hljs-string">"page"</span>,
- <span class="hljs-string">"diffUrl"</span>,
- <span class="hljs-string">"added"</span>,
- <span class="hljs-string">"comment"</span>,
- <span class="hljs-string">"commentLength"</span>,
- <span class="hljs-string">"isNew"</span>,
- <span class="hljs-string">"isMinor"</span>,
- <span class="hljs-string">"delta"</span>,
- <span class="hljs-string">"isAnonymous"</span>,
- <span class="hljs-string">"user"</span>,
- <span class="hljs-string">"deltaBucket"</span>,
- <span class="hljs-string">"deleted"</span>,
- <span class="hljs-string">"namespace"</span>,
- <span class="hljs-string">"cityName"</span>,
- <span class="hljs-string">"countryName"</span>,
- <span class="hljs-string">"regionIsoCode"</span>,
- <span class="hljs-string">"metroCode"</span>,
- <span class="hljs-string">"countryIsoCode"</span>,
- <span class="hljs-string">"regionName"</span>
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">source</span>
-PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre>
-<p></details></p></li>
-<li><p>Review the generated SQL query to make sure it matches your requirements and does what you expect.</p></li>
-<li><p>Click <strong>Run</strong> to start the ingestion.</p></li>
-</ol>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/connect-external-data.html"><span class="arrow-prev">← </span><span>Tutorial - Load files with SQL-based ingestion</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/examples.html"><span>Examples</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><d [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/multi-stage-query/examples.html b/docs/24.0.0/multi-stage-query/examples.html
index 079adb3b..eb800e68 100644
--- a/docs/24.0.0/multi-stage-query/examples.html
+++ b/docs/24.0.0/multi-stage-query/examples.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-example-queries.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion query examples</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/examples.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion query examples</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,9 +78,11 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>These example queries show you some of the things you can do when modifying queries for your use case. Copy the example queries into the <strong>Query</strong> view of the Druid console and run them to see what they do.</p>
+<p>These example queries show you some of the things you can do when modifying queries for your use case. Copy the example queries into the <strong>Query</strong> view of the web console and run them to see what they do.</p>
<h2><a class="anchor" aria-hidden="true" id="insert-with-no-rollup"></a><a href="#insert-with-no-rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
<p>This example inserts data into a table named <code>w000</code> without performing any data rollup:</p>
<p><details><summary>Show the query</summary></p>
@@ -122,7 +124,7 @@ CLUSTERED <span class="hljs-keyword">BY</span> channel
</code></pre>
<p></details></p>
<h2><a class="anchor" aria-hidden="true" id="insert-with-rollup"></a><a href="#insert-with-rollup" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>This example inserts data into a table named <code>kttm_data</code> and performs data rollup. This example implements the recommendations described in <a href="/docs/24.0.0/multi-stage-query/index.html#multi-value-dimensions">multi-value dimensions</a>.</p>
+<p>This example inserts data into a table named <code>kttm_data</code> and performs data rollup. This example implements the recommendations described in <a href="/docs/24.0.0/multi-stage-query/concepts.html#rollup">Rollup</a>.</p>
<p><details><summary>Show the query</summary></p>
<pre><code class="hljs css language-sql"><span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> <span class="hljs-string">"kttm_rollup"</span>
@@ -478,12 +480,7 @@ L_STATE_FIPS <span class="hljs-keyword">AS</span> (
<span class="hljs-keyword">LIMIT</span> <span class="hljs-number">1000</span>
</code></pre>
<p></details></p>
-<h2><a class="anchor" aria-hidden="true" id="next-steps"></a><a href="#next-steps" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<ul>
-<li><a href="/docs/24.0.0/multi-stage-query/examples.html">Read Multi-stage queries</a> to learn more about how multi-stage queries work.</li>
-<li><a href="/docs/24.0.0/operations/druid-console.html">Explore the Query view</a> to learn about the UI tools to help you get started.</li>
-</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/convert-json-spec.html"><span class="arrow-prev">← </span><span>Tutorial - Convert an ingestion spec for SQL-based ingestion</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/api.html"><span>API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#insert-with-no-rollup">INSE [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/security.html"><span class="arrow-prev">← </span><span>Security</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/reference.html"><span>Reference</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#insert-with-no-rollup">INSERT with no rollup</a></li><li><a href="#insert-wi [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/index.html b/docs/24.0.0/multi-stage-query/index.html
index 586f4778..9a94db7e 100644
--- a/docs/24.0.0/multi-stage-query/index.html
+++ b/docs/24.0.0/multi-stage-query/index.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>SQL-based ingestion overview and syntax · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Introduces multi-stage query architecture and its task engine"/><meta name="docsea [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>SQL-based ingestion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/multi-stage-query/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Introduces multi-stage query architecture and its task engine"/><meta name="docsearch:language" conten [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion overview and syntax</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,262 +78,45 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>SQL-based ingestion for Apache Druid uses a distributed multi-stage query architecture, which includes a query engine called the multi-stage query task engine (MSQ task engine). The MSQ task engine extends Druid's query capabilities, so you can write queries that reference <a href="#read-external-data">external data</a> as well as perform ingestion with SQL <a href="#insert-data">INSERT</a> and <a href="#replace-data">REPLACE</a>. Essentially, you can perform SQL-based ingestion inste [...]
-<p>The MSQ task engine excels at executing queries that can get bottlenecked at the Broker when using Druid's native SQL engine. When you submit queries, the MSQ task engine splits them into stages and automatically exchanges data between stages. Each stage is parallelized to run across multiple data servers at once, simplifying performance.</p>
-<h2><a class="anchor" aria-hidden="true" id="msq-task-engine-features"></a><a href="#msq-task-engine-features" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-. [...]
-<p>In its current state, the MSQ task engine enables you to do the following:</p>
+<p>Apache Druid supports SQL-based ingestion using the bundled <a href="#load-the-extension"><code>druid-multi-stage-query</code> extension</a>.
+This extension adds a <a href="/docs/24.0.0/multi-stage-query/concepts.html#sql-task-engine">multi-stage query task engine for SQL</a> that allows running SQL
+<a href="/docs/24.0.0/multi-stage-query/concepts.html#insert">INSERT</a> and <a href="/docs/24.0.0/multi-stage-query/concepts.html#replace">REPLACE</a> statements as batch tasks.</p>
+<p>Nearly all SELECT capabilities are available for <code>INSERT ... SELECT</code> and <code>REPLACE ... SELECT</code> queries, with certain
+exceptions listed on the <a href="/docs/24.0.0/multi-stage-query/known-issues.html#select">Known issues</a> page. This allows great flexibility to apply
+transformations, filters, JOINs, aggregations, and so on while ingesting data. This also allows in-database
+transformation: creating new tables based on queries of other tables.</p>
+<h2><a class="anchor" aria-hidden="true" id="vocabulary"></a><a href="#vocabulary" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<ul>
-<li>Read external data at query time using EXTERN.</li>
-<li>Execute batch ingestion jobs by writing SQL queries using INSERT and REPLACE. You no longer need to generate a JSON-based ingestion spec.</li>
-<li>Transform and rewrite existing tables using SQL.</li>
-<li>Perform multi-dimension range partitioning reliably, which leads to more evenly distributed segment sizes and better performance.</li>
-</ul>
-<p>The MSQ task engine has additional features that can be used as part of a proof of concept or demo, but don't use or rely on the following features for any meaningful use cases, especially production ones:</p>
-<ul>
-<li>Execute heavy-weight queries and return large numbers of rows.</li>
-<li>Execute queries that exchange large amounts of data between servers, like exact count distinct of high-cardinality fields.</li>
+<li><p><strong>Controller</strong>: An indexing service task of type <code>query_controller</code> that manages
+the execution of a query. There is one controller task per query.</p></li>
+<li><p><strong>Worker</strong>: Indexing service tasks of type <code>query_worker</code> that execute a
+query. There can be multiple worker tasks per query. Internally,
+the tasks process items in parallel using their processing pools (up to <code>druid.processing.numThreads</code> of execution parallelism
+within a worker task).</p></li>
+<li><p><strong>Stage</strong>: A stage of query execution that is parallelized across
+worker tasks. Workers exchange data with each other between stages.</p></li>
+<li><p><strong>Partition</strong>: A slice of data output by worker tasks. In INSERT or REPLACE
+queries, the partitions of the final stage become Druid segments.</p></li>
+<li><p><strong>Shuffle</strong>: Workers exchange data between themselves on a per-partition basis in a process called
+shuffling. During a shuffle, each output partition is sorted by a clustering key.</p></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="load-the-extension"></a><a href="#load-the-extension" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>For new clusters that use 24.0 or later, the multi-stage query extension is loaded by default. If you want to add the extension to an existing cluster, add the extension <code>druid-multi-stage-query</code> to <code>druid.extensions.loadlist</code> in your <code>common.runtime.properties</code> file.</p>
+<p>To add the extension to an existing cluster, add <code>druid-multi-stage-query</code> to <code>druid.extensions.loadlist</code> in your
+<code>common.runtime.properties</code> file.</p>
<p>For more information about how to load an extension, see <a href="/docs/24.0.0/development/extensions.html#loading-extensions">Loading extensions</a>.</p>
-<p>To use EXTERN, you need READ permission on the resource named "EXTERNAL" of the resource type "EXTERNAL". If you encounter a 403 error when trying to use EXTERN, verify that you have the correct permissions.</p>
-<h2><a class="anchor" aria-hidden="true" id="msq-task-engine-query-syntax"></a><a href="#msq-task-engine-query-syntax" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
-<p>You can submit queries to the MSQ task engine through the <strong>Query</strong> view in the Druid console or through the API. The Druid console is a good place to start because you can preview a query before you run it. You can also experiment with many of the <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters">context parameters</a> through the UI. Once you're comfortable with submitting queries through the Druid console, <a href="/docs/24.0.0/multi-stage-quer [...]
-<p>If you encounter an issue after you submit a query, you can learn more about what an error means from the <a href="/docs/24.0.0/multi-stage-query/concepts.html#limits">limits</a> and <a href="/docs/24.0.0/multi-stage-query/concepts.html#error-codes">errors</a>.</p>
-<p>Queries for the MSQ task engine involve three primary functions:</p>
-<ul>
-<li>EXTERN to query external data</li>
-<li>INSERT INTO ... SELECT to insert data, such as data from an external source</li>
-<li>REPLACE to replace existing datasources, partially or fully, with query results</li>
-</ul>
-<p>For information about the syntax for queries, see <a href="/docs/24.0.0/multi-stage-query/reference.html#sql-syntax">SQL syntax</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="read-external-data"></a><a href="#read-external-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>Query tasks can access external data through the EXTERN function. When using EXTERN, keep in mind that large files do not get split across different worker tasks. If you have fewer input files than worker tasks, you can increase query parallelism by splitting up your input files such that you have at least one input file per worker task.</p>
-<p>You can use the EXTERN function anywhere a table is expected in the following form: <code>TABLE(EXTERN(...))</code>. You can use external data with SELECT, INSERT, and REPLACE queries.</p>
-<p>The following query reads external data:</p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">SELECT</span>
- *
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type": "json"}'</span>,
- <span class="hljs-string">'[{"name": "timestamp", "type": "string"}, {"name": "page", "type": "string"}, {"name": "user", "type": "string"}]'</span>
- )
-)
-<span class="hljs-keyword">LIMIT</span> <span class="hljs-number">100</span>
-</code></pre>
-<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#extern">EXTERN</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="insert-data"></a><a href="#insert-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>With the MSQ task engine, Druid can use the results of a query task to create a new datasource or to append to an existing datasource. Syntactically, there is no difference between the two. These operations use the INSERT INTO ... SELECT syntax.</p>
-<p>All SELECT capabilities are available for INSERT queries. However, the MSQ task engine does not include all the existing SQL query features of Druid. See <a href="/docs/24.0.0/multi-stage-query/known-issues.html">Known issues</a> for a list of capabilities that aren't available.</p>
-<p>The following example query inserts data from an external source into a table named <code>w000</code> and partitions it by day:</p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> w000
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <span class="hljs-string">"page"</span>,
- <span class="hljs-string">"user"</span>
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type": "json"}'</span>,
- <span class="hljs-string">'[{"name": "timestamp", "type": "string"}, {"name": "page", "type": "string"}, {"name": "user", "type": "string"}]'</span>
- )
-)
-PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre>
-<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#insert">INSERT</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="replace-data"></a><a href="#replace-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
-<p>The syntax for REPLACE is similar to INSERT. All SELECT functionality is available for REPLACE queries.
-Note that the MSQ task engine does not yet implement all native Druid query features.
-For details, see <a href="/docs/24.0.0/multi-stage-query/known-issues.html">Known issues</a>.</p>
-<p>When working with REPLACE queries, keep the following in mind:</p>
-<ul>
-<li>The intervals generated as a result of the OVERWRITE WHERE query must align with the granularity specified in the PARTITIONED BY clause.</li>
-<li>OVERWRITE WHERE queries only support the <code>__time</code> column.</li>
-</ul>
-<p>For more information about the syntax, see <a href="/docs/24.0.0/multi-stage-query/reference.html#replace">REPLACE</a>.</p>
-<p>The following examples show how to replace data in a table.</p>
-<h4><a class="anchor" aria-hidden="true" id="replace-all-data"></a><a href="#replace-all-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
-<p>You can replace all the data in a table by using REPLACE INTO ... OVERWRITE ALL SELECT:</p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> w000
-OVERWRITE <span class="hljs-keyword">ALL</span>
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <span class="hljs-string">"page"</span>,
- <span class="hljs-string">"user"</span>
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type": "json"}'</span>,
- <span class="hljs-string">'[{"name": "timestamp", "type": "string"}, {"name": "page", "type": "string"}, {"name": "user", "type": "string"}]'</span>
- )
-)
-PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre>
-<h4><a class="anchor" aria-hidden="true" id="replace-some-data"></a><a href="#replace-some-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
-<p>You can replace some of the data in a table by using REPLACE INTO ... OVERWRITE WHERE ... SELECT:</p>
-<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> w000
-OVERWRITE <span class="hljs-keyword">WHERE</span> __time >= <span class="hljs-built_in">TIMESTAMP</span> <span class="hljs-string">'2019-08-25'</span> <span class="hljs-keyword">AND</span> __time < <span class="hljs-built_in">TIMESTAMP</span> <span class="hljs-string">'2019-08-28'</span>
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <span class="hljs-string">"page"</span>,
- <span class="hljs-string">"user"</span>
-<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
- EXTERN(
- <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
- <span class="hljs-string">'{"type": "json"}'</span>,
- <span class="hljs-string">'[{"name": "timestamp", "type": "string"}, {"name": "page", "type": "string"}, {"name": "user", "type": "string"}]'</span>
- )
-)
-PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
-</code></pre>
-<h2><a class="anchor" aria-hidden="true" id="adjust-query-behavior"></a><a href="#adjust-query-behavior" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
-<p>In addition to the basic functions, you can further modify your query behavior to control how your queries run or what your results look like. You can control how your queries behave by changing the following:</p>
-<h3><a class="anchor" aria-hidden="true" id="primary-timestamp"></a><a href="#primary-timestamp" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
-<p>Druid tables always include a primary timestamp named <code>__time</code>, so your ingestion query should generally include a column named <code>__time</code>.</p>
-<p>The following formats are supported for <code>__time</code> in the source data:</p>
-<ul>
-<li>ISO 8601 with 'T' separator, such as "2000-01-01T01:02:03.456"</li>
-<li>Milliseconds since Unix epoch (00:00:00 UTC on January 1, 1970)</li>
-</ul>
-<p>The <code>__time</code> column is used for time-based partitioning, such as <code>PARTITIONED BY DAY</code>.</p>
-<p>If you use <code>PARTITIONED BY ALL</code> or <code>PARTITIONED BY ALL TIME</code>, time-based
-partitioning is disabled. In these cases, your ingestion query doesn't need
-to include a <code>__time</code> column. However, Druid still creates a <code>__time</code> column
-in your Druid table and sets all timestamps to 1970-01-01 00:00:00.</p>
-<p>For more information, see <a href="/docs/24.0.0/ingestion/data-model.html#primary-timestamp">Primary timestamp</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="partitioned-by"></a><a href="#partitioned-by" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
-<p>INSERT and REPLACE queries require the PARTITIONED BY clause, which determines how time-based partitioning is done. In Druid, data is split into segments, one or more per time chunk defined by the PARTITIONED BY granularity. A good general rule is to adjust the granularity so that each segment contains about five million rows. Choose a granularity based on your ingestion rate. For example, if you ingest a million rows per day, PARTITIONED BY DAY is good. If you ingest a million rows a [...]
-<p>Using the clause provides the following benefits:</p>
-<ul>
-<li>Better query performance due to time-based segment pruning, which removes segments from
-consideration when they do not contain any data for a query's time filter.</li>
-<li>More efficient data management, as data can be rewritten for each time partition individually
-rather than the whole table.</li>
-</ul>
-<p>You can use the following arguments for PARTITIONED BY:</p>
-<ul>
-<li>Time unit: <code>HOUR</code>, <code>DAY</code>, <code>MONTH</code>, or <code>YEAR</code>. Equivalent to <code>FLOOR(__time TO TimeUnit)</code>.</li>
-<li><code>TIME_FLOOR(__time, 'granularity_string')</code>, where granularity_string is an ISO 8601 period like
-'PT1H'. The first argument must be <code>__time</code>.</li>
-<li><code>FLOOR(__time TO TimeUnit)</code>, where <code>TimeUnit</code> is any unit supported by the <a href="/docs/24.0.0/querying/sql-scalar.html#date-and-time-functions">FLOOR function</a>. The first
-argument must be <code>__time</code>.</li>
-<li><code>ALL</code> or <code>ALL TIME</code>, which effectively disables time partitioning by placing all data in a single
-time chunk. To use LIMIT or OFFSET at the outer level of your INSERT or REPLACE query, you must set PARTITIONED BY to ALL or ALL TIME.</li>
-</ul>
-<p>You can use the following ISO 8601 periods for <code>TIME_FLOOR</code>:</p>
-<ul>
-<li>PT1S</li>
-<li>PT1M</li>
-<li>PT5M</li>
-<li>PT10M</li>
-<li>PT15M</li>
-<li>PT30M</li>
-<li>PT1H</li>
-<li>PT6H</li>
-<li>P1D</li>
-<li>P1W</li>
-<li>P1M</li>
-<li>P3M</li>
-<li>P1Y</li>
-</ul>
-<h3><a class="anchor" aria-hidden="true" id="clustered-by"></a><a href="#clustered-by" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
-<p>Data is first divided by the PARTITIONED BY clause. Data can be further split by the CLUSTERED BY clause. For example, suppose you ingest 100 M rows per hour and use <code>PARTITIONED BY HOUR</code> as your time partition. You then divide up the data further by adding <code>CLUSTERED BY hostName</code>. The result is segments of about 5 million rows, with like <code>hostName</code>s grouped within the same segment.</p>
-<p>Using CLUSTERED BY has the following benefits:</p>
-<ul>
-<li>Lower storage footprint due to combining similar data into the same segments, which improves
-compressibility.</li>
-<li>Better query performance due to dimension-based segment pruning, which removes segments from
-consideration when they cannot possibly contain data matching a query's filter.</li>
-</ul>
-<p>For dimension-based segment pruning to be effective, your queries should meet the following conditions:</p>
-<ul>
-<li>All CLUSTERED BY columns are single-valued string columns</li>
-<li>Use a REPLACE query for ingestion</li>
-</ul>
-<p>Druid still clusters data during ingestion if these conditions aren't met but won't perform dimension-based segment pruning at query time. That means if you use an INSERT query for ingestion or have numeric columns or multi-valued string columns, dimension-based segment pruning doesn't occur at query time.</p>
-<p>You can tell if dimension-based segment pruning is possible by using the <code>sys.segments</code> table to
-inspect the <code>shard_spec</code> for the segments generated by an ingestion query. If they are of type
-<code>range</code> or <code>single</code>, then dimension-based segment pruning is possible. Otherwise, it is not. The
-shard spec type is also available in the <strong>Segments</strong> view under the <strong>Partitioning</strong>
-column.</p>
-<p>You can use the following filters for dimension-based segment pruning:</p>
-<ul>
-<li>Equality to string literals, like <code>x = 'foo'</code> or <code>x IN ('foo', 'bar')</code>.</li>
-<li>Comparison to string literals, like <code>x < 'foo'</code> or other comparisons involving <code><</code>, <code>></code>, <code><=</code>, or <code>>=</code>.</li>
-</ul>
-<p>This differs from multi-dimension range based partitioning in classic batch ingestion where both
-string and numeric columns support Broker-level pruning. With SQL-based batch ingestion,
-only string columns support Broker-level pruning.</p>
-<p>It is okay to mix time partitioning with secondary partitioning. For example, you can
-combine <code>PARTITIONED BY HOUR</code> with <code>CLUSTERED BY channel</code> to perform
-time partitioning by hour and secondary partitioning by channel within each hour.</p>
-<h3><a class="anchor" aria-hidden="true" id="group-by"></a><a href="#group-by" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
-<p>A query's GROUP BY clause determines how data is rolled up. The expressions in the GROUP BY clause become
-dimensions, and aggregation functions become metrics.</p>
-<h3><a class="anchor" aria-hidden="true" id="ingest-time-aggregations"></a><a href="#ingest-time-aggregations" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-. [...]
-<p>When performing rollup using aggregations, it is important to use aggregators
-that return nonfinalized state. This allows you to perform further rollups
-at query time. To achieve this, set <code>finalizeAggregations: false</code> in your
-ingestion query context.</p>
-<p>Check out the <a href="/docs/24.0.0/multi-stage-query/examples.html#insert-with-rollup">INSERT with rollup example query</a> to see this feature in
-action.</p>
-<p>Druid needs information for aggregating measures of different segments to compact. For example, to aggregate <code>count("col") as example_measure</code>, Druid needs to sum the value of <code>example_measure</code>
-across the segments. This information is stored inside the metadata of the segment. For the SQL-based ingestion, Druid only populates the
-aggregator information of a column in the segment metadata when:</p>
-<ul>
-<li>The INSERT or REPLACE query has an outer GROUP BY clause.</li>
-<li>The following context parameters are set for the query context: <code>finalizeAggregations: false</code> and <code>groupByEnableMultiValueUnnesting: false</code></li>
-</ul>
-<p>The following table lists query-time aggregations for SQL-based ingestion:</p>
-<table>
-<thead>
-<tr><th>Query-time aggregation</th><th>Notes</th></tr>
-</thead>
-<tbody>
-<tr><td>SUM</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>MIN</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>MAX</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>AVG</td><td>Use SUM and COUNT at ingest time. Switch to quotient of SUM at query time.</td></tr>
-<tr><td>COUNT</td><td>Use unchanged at ingest time, but switch to SUM at query time.</td></tr>
-<tr><td>COUNT(DISTINCT expr)</td><td>If approximate, use APPROX_COUNT_DISTINCT at ingest time.<br /><br />If exact, you cannot use an ingest-time aggregation. Instead, <code>expr</code> must be stored as-is. Add it to the SELECT and GROUP BY lists.</td></tr>
-<tr><td>EARLIEST(expr)<br /><br />(numeric form)</td><td>Not supported.</td></tr>
-<tr><td>EARLIEST(expr, maxBytes)<br /><br />(string form)</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>LATEST(expr)<br /><br />(numeric form)</td><td>Not supported.</td></tr>
-<tr><td>LATEST(expr, maxBytes)<br /><br />(string form)</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>APPROX_COUNT_DISTINCT</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>APPROX_COUNT_DISTINCT_BUILTIN</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>APPROX_COUNT_DISTINCT_DS_HLL</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>APPROX_COUNT_DISTINCT_DS_THETA</td><td>Use unchanged at ingest time.</td></tr>
-<tr><td>APPROX_QUANTILE</td><td>Not supported. Deprecated; use APPROX_QUANTILE_DS instead.</td></tr>
-<tr><td>APPROX_QUANTILE_DS</td><td>Use DS_QUANTILES_SKETCH at ingest time. Continue using APPROX_QUANTILE_DS at query time.</td></tr>
-<tr><td>APPROX_QUANTILE_FIXED_BUCKETS</td><td>Not supported.</td></tr>
-</tbody>
-</table>
-<h3><a class="anchor" aria-hidden="true" id="multi-value-dimensions"></a><a href="#multi-value-dimensions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
-<p>By default, multi-value dimensions are not ingested as expected when rollup is enabled because the
-GROUP BY operator unnests them instead of leaving them as arrays. This is <a href="/docs/24.0.0/querying/sql-data-types.html#multi-value-strings">standard behavior</a> for GROUP BY but it is generally not desirable behavior for ingestion.</p>
-<p>To address this:</p>
-<ul>
-<li>When using GROUP BY with data from EXTERN, wrap any string type fields from EXTERN that may be
-multi-valued in <code>MV_TO_ARRAY</code>.</li>
-<li>Set <code>groupByEnableMultiValueUnnesting: false</code> in your query context to ensure that all multi-value
-strings are properly converted to arrays using <code>MV_TO_ARRAY</code>. If any strings aren't
-wrapped in <code>MV_TO_ARRAY</code>, the query reports an error that includes the message "Encountered
-multi-value dimension x that cannot be processed with groupByEnableMultiValueUnnesting set to false."</li>
-</ul>
-<p>For an example, see <a href="/docs/24.0.0/multi-stage-query/examples.html#insert-with-rollup">INSERT with rollup example query</a>.</p>
-<h3><a class="anchor" aria-hidden="true" id="context-parameters"></a><a href="#context-parameters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>Context parameters can control things such as how many tasks get launched or what happens if there's a malformed record.</p>
-<p>For a full list of context parameters and how they affect a query, see <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters">Context parameters</a>.</p>
+<p>To use <a href="/docs/24.0.0/multi-stage-query/reference.html#extern">EXTERN</a>, you need READ permission on the resource named "EXTERNAL" of the resource type
+"EXTERNAL". If you encounter a 403 error when trying to use EXTERN, verify that you have the correct permissions.</p>
<h2><a class="anchor" aria-hidden="true" id="next-steps"></a><a href="#next-steps" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<ul>
-<li><a href="/docs/24.0.0/multi-stage-query/concepts.html">Understand how the multi-stage query architecture works</a> by reading about the concepts behind it and its processes.</li>
-<li><a href="/docs/24.0.0/operations/druid-console.html">Explore the Query view</a> to learn about the UI tools that can help you get started.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/concepts.html">Read about key concepts</a> to learn more about how SQL-based ingestion and multi-stage queries work.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/examples.html">Check out the examples</a> to see SQL-based ingestion in action.</li>
+<li><a href="/docs/24.0.0/operations/web-console.html">Explore the Query view</a> to get started in the web console.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/faq.html"><span class="arrow-prev">← </span><span>Troubleshooting FAQ</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/concepts.html"><span>Key concepts</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#msq-task-engine-features">MSQ task engine features</a></li><li><a href="#loa [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/ingestion/hadoop.html"><span class="arrow-prev">← </span><span>Hadoop-based</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/concepts.html"><span>Key concepts</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#vocabulary">Vocabulary</a></li><li><a href="#load-the-extension">Load the extens [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/known-issues.html b/docs/24.0.0/multi-stage-query/known-issues.html
index e198de7b..0e6ff91d 100644
--- a/docs/24.0.0/multi-stage-query/known-issues.html
+++ b/docs/24.0.0/multi-stage-query/known-issues.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-known-issues.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion known issues</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/known-issues.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion known issues</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,20 +78,22 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0.
-Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported.
-We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment
-before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statements that do not
-write to a datasource is experimental.</p>
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
<h2><a class="anchor" aria-hidden="true" id="multi-stage-query-task-runtime"></a><a href="#multi-stage-query-task-runtime" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
<ul>
<li><p>Fault tolerance is not implemented. If any task fails, the entire query fails.</p></li>
-<li><p>SELECT from a Druid datasource does not include unpublished real-time data.</p></li>
-<li><p>GROUPING SETS is not implemented. Queries that use GROUPING SETS fail.</p></li>
<li><p>Worker task stage outputs are stored in the working directory given by <code>druid.indexer.task.baseDir</code>. Stages that
generate a large amount of output data may exhaust all available disk space. In this case, the query fails with
an <a href="/docs/24.0.0/multi-stage-query/reference.html#error-codes">UnknownError</a> with a message including "No space left on device".</p></li>
+</ul>
+<h2><a class="anchor" aria-hidden="true" id="select"></a><a href="#select" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
+<ul>
+<li><p>SELECT from a Druid datasource does not include unpublished real-time data.</p></li>
+<li><p>GROUPING SETS and UNION ALL are not implemented. Queries using these features return a
+<a href="/docs/24.0.0/multi-stage-query/reference.html#error-codes">QueryNotSupported</a> error.</p></li>
<li><p>The numeric varieties of the EARLIEST and LATEST aggregators do not work properly. Attempting to use the numeric
varieties of these aggregators lead to an error like
<code>java.lang.ClassCastException: class java.lang.Double cannot be cast to class org.apache.druid.collections.SerializablePair</code>.
@@ -99,9 +101,12 @@ The string varieties, however, do work properly.</p></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="insert-and-replace"></a><a href="#insert-and-replace" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
<ul>
-<li><p>INSERT with column lists, like <code>INSERT INTO tbl (a, b, c) SELECT ...</code>, is not implemented.</p></li>
-<li><p><code>INSERT ... SELECT</code> inserts columns from the SELECT statement based on column name. This differs from SQL standard
-behavior, where columns are inserted based on position.</p></li>
+<li><p>INSERT and REPLACE with column lists, like <code>INSERT INTO tbl (a, b, c) SELECT ...</code>, is not implemented.</p></li>
+<li><p><code>INSERT ... SELECT</code> and <code>REPLACE ... SELECT</code> insert columns from the SELECT statement based on column name. This
+differs from SQL standard behavior, where columns are inserted based on position.</p></li>
+<li><p>INSERT and REPLACE do not support all options available in <a href="/docs/24.0.0/ingestion/ingestion-spec.html">ingestion specs</a>,
+including the <code>createBitmapIndex</code> and <code>multiValueHandling</code> <a href="/docs/24.0.0/ingestion/ingestion-spec.html#dimension-objects">dimension</a>
+properties, and the <code>indexSpec</code> <a href="/docs/24.0.0/ingestion/ingestion-spec.html#tuningconfig"><code>tuningConfig</code></a> property.</p></li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="extern"></a><a href="#extern" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
<ul>
@@ -111,7 +116,7 @@ of the <a href="/docs/24.0.0/multi-stage-query/reference.html#extern">EXTERN fun
<li><p>EXTERN with input sources that match large numbers of files may exhaust available memory on the controller task.</p></li>
<li><p>EXTERN does not accept <code>druid</code> input sources. Use FROM instead.</p></li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/reference.html"><span class="arrow-prev">← </span><span>Reference</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql.html"><span>Overview and syntax</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#multi-stage-query-task-runtime">Multi-stage query task runtime</a></li><li><a h [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/reference.html"><span class="arrow-prev">← </span><span>Reference</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/tasks.html"><span>Task reference</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#multi-stage-query-task-runtime">Multi-stage query task runtime</a></li><li><a hre [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/reference.html b/docs/24.0.0/multi-stage-query/reference.html
index 7b183e2c..7620ab4e 100644
--- a/docs/24.0.0/multi-stage-query/reference.html
+++ b/docs/24.0.0/multi-stage-query/reference.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-reference.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion reference</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/reference.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion reference</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,48 +78,13 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>This topic is a reference guide for the multi-stage query architecture in Apache Druid.</p>
-<h2><a class="anchor" aria-hidden="true" id="context-parameters"></a><a href="#context-parameters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
-<p>In addition to the Druid SQL <a href="/docs/24.0.0/querying/sql-query-context.html">context parameters</a>, the multi-stage query task engine accepts certain context parameters that are specific to it.</p>
-<p>Use context parameters alongside your queries to customize the behavior of the query. If you're using the API, include the context parameters in the query context when you submit a query:</p>
-<pre><code class="hljs css language-json">{
- <span class="hljs-attr">"query"</span>: <span class="hljs-string">"SELECT 1 + 1"</span>,
- <span class="hljs-attr">"context"</span>: {
- <span class="hljs-attr">"<key>"</span>: <span class="hljs-string">"<value>"</span>,
- <span class="hljs-attr">"maxNumTasks"</span>: <span class="hljs-number">3</span>
- }
-}
-</code></pre>
-<p>If you're using the Druid console, you can specify the context parameters through various UI options.</p>
-<p>The following table lists the context parameters for the MSQ task engine:</p>
-<table>
-<thead>
-<tr><th>Parameter</th><th>Description</th><th>Default value</th></tr>
-</thead>
-<tbody>
-<tr><td>maxNumTasks</td><td>SELECT, INSERT, REPLACE<br /><br />The maximum total number of tasks to launch, including the controller task. The lowest possible value for this setting is 2: one controller and one worker. All tasks must be able to launch simultaneously. If they cannot, the query returns a <code>TaskStartTimeout</code> error code after approximately 10 minutes.<br /><br />May also be provided as <code>numTasks</code>. If both are present, <code>maxNumTasks</code> takes prior [...]
-<tr><td>taskAssignment</td><td>SELECT, INSERT, REPLACE<br /><br />Determines how many tasks to use. Possible values include: <ul><li><code>max</code>: Uses as many tasks as possible, up to <code>maxNumTasks</code>.</li><li><code>auto</code>: When file sizes can be determined through directory listing (for example: local files, S3, GCS, HDFS) uses as few tasks as possible without exceeding 10 GiB or 10,000 files per task, unless exceeding these limits is necessary to stay within <code>max [...]
-<tr><td>finalizeAggregations</td><td>SELECT, INSERT, REPLACE<br /><br />Determines the type of aggregation to return. If true, Druid finalizes the results of complex aggregations that directly appear in query results. If false, Druid returns the aggregation's intermediate type rather than finalized type. This parameter is useful during ingestion, where it enables storing sketches directly in Druid tables. For more information about aggregations, see <a href="/docs/24.0.0/querying/sql-agg [...]
-<tr><td>rowsInMemory</td><td>INSERT or REPLACE<br /><br />Maximum number of rows to store in memory at once before flushing to disk during the segment generation process. Ignored for non-INSERT queries. In most cases, use the default value. You may need to override the default if you run into one of the <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> around memory usage.</td><td>100,000</td></tr>
-<tr><td>segmentSortOrder</td><td>INSERT or REPLACE<br /><br />Normally, Druid sorts rows in individual segments using <code>__time</code> first, followed by the <a href="/docs/24.0.0/multi-stage-query/index.html#clustered-by">CLUSTERED BY</a> clause. When you set <code>segmentSortOrder</code>, Druid sorts rows in segments using this column list first, followed by the CLUSTERED BY order.<br /><br />You provide the column list as comma-separated values or as a JSON array in string form. If [...]
-<tr><td>maxParseExceptions</td><td>SELECT, INSERT, REPLACE<br /><br />Maximum number of parse exceptions that are ignored while executing the query before it stops with <code>TooManyWarningsFault</code>. To ignore all the parse exceptions, set the value to -1.</td><td>0</td></tr>
-<tr><td>rowsPerSegment</td><td>INSERT or REPLACE<br /><br />The number of rows per segment to target. The actual number of rows per segment may be somewhat higher or lower than this number. In most cases, use the default. For general information about sizing rows per segment, see <a href="/docs/24.0.0/operations/segment-optimization.html">Segment Size Optimization</a>.</td><td>3,000,000</td></tr>
-<tr><td>sqlTimeZone</td><td>Sets the time zone for this connection, which affects how time functions and timestamp literals behave. Use a time zone name like "America/Los_Angeles" or offset like "-08:00".</td><td><code>druid.sql.planner.sqlTimeZone</code> on the Broker (default: UTC)</td></tr>
-<tr><td>useApproximateCountDistinct</td><td>Whether to use an approximate cardinality algorithm for <code>COUNT(DISTINCT foo)</code>.</td><td><code>druid.sql.planner.useApproximateCountDistinct</code> on the Broker (default: true)</td></tr>
-</tbody>
-</table>
-<h2><a class="anchor" aria-hidden="true" id="error-codes"></a><a href="#error-codes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>Error codes have corresponding human-readable messages that explain the error. For more information about the error codes, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#error-codes">Error codes</a>.</p>
-<h2><a class="anchor" aria-hidden="true" id="sql-syntax"></a><a href="#sql-syntax" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
-<p>The MSQ task engine has three primary SQL functions:</p>
-<ul>
-<li>EXTERN</li>
-<li>INSERT</li>
-<li>REPLACE</li>
-</ul>
-<p>For information about using these functions and their corresponding examples, see <a href="/docs/24.0.0/multi-stage-query/index.html#msq-task-engine-query-syntax">MSQ task engine query syntax</a>. For information about adjusting the shape of your data, see <a href="/docs/24.0.0/multi-stage-query/index.html#adjust-query-behavior">Adjust query behavior</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="sql-reference"></a><a href="#sql-reference" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
+<p>This topic is a reference guide for the multi-stage query architecture in Apache Druid. For examples of real-world
+usage, refer to the <a href="/docs/24.0.0/multi-stage-query/examples.html">Examples</a> page.</p>
<h3><a class="anchor" aria-hidden="true" id="extern"></a><a href="#extern" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
<p>Use the EXTERN function to read external data.</p>
<p>Function format:</p>
@@ -139,65 +104,184 @@
<li>Any <a href="/docs/24.0.0/ingestion/data-formats.html">Druid input format</a> as a JSON-encoded string.</li>
<li>A row signature, as a JSON-encoded array of column descriptors. Each column descriptor must have a <code>name</code> and a <code>type</code>. The type can be <code>string</code>, <code>long</code>, <code>double</code>, or <code>float</code>. This row signature is used to map the external data into the SQL layer.</li>
</ol>
+<p>For more information, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#extern">Read external data with EXTERN</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="insert"></a><a href="#insert" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
-<p>Use the INSERT function to insert data.</p>
-<p>Unlike standard SQL, INSERT inserts data according to column name and not positionally. This means that it is important for the output column names of subsequent INSERT queries to be the same as the table. Do not rely on their positions within the SELECT clause.</p>
-<p>Function format:</p>
+<p>Use the INSERT statement to insert data.</p>
+<p>Unlike standard SQL, INSERT loads data into the target table according to column name, not positionally. If necessary,
+use <code>AS</code> in your SELECT column list to assign the correct names. Do not rely on their positions within the SELECT
+clause.</p>
+<p>Statement format:</p>
<pre><code class="hljs css language-sql"><span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> <<span class="hljs-keyword">table</span> <span class="hljs-keyword">name</span>>
-<span class="hljs-keyword">SELECT</span>
- <<span class="hljs-keyword">column</span>>
-<span class="hljs-keyword">FROM</span> <<span class="hljs-keyword">table</span>>
+< <span class="hljs-keyword">SELECT</span> <span class="hljs-keyword">query</span> >
PARTITIONED <span class="hljs-keyword">BY</span> <<span class="hljs-built_in">time</span> frame>
+[ CLUSTERED <span class="hljs-keyword">BY</span> <<span class="hljs-keyword">column</span> <span class="hljs-keyword">list</span>> ]
</code></pre>
<p>INSERT consists of the following parts:</p>
<ol>
<li>Optional <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters">context parameters</a>.</li>
<li>An <code>INSERT INTO <dataSource></code> clause at the start of your query, such as <code>INSERT INTO your-table</code>.</li>
-<li>A clause for the data you want to insert, such as <code>SELECT...FROM TABLE...</code>. You can use EXTERN to reference external tables using the following format: ``TABLE(EXTERN(...))`.</li>
-<li>A <a href="/docs/24.0.0/multi-stage-query/index.html#partitioned-by">PARTITIONED BY</a> clause for your INSERT statement. For example, use PARTITIONED BY DAY for daily partitioning or PARTITIONED BY ALL TIME to skip time partitioning completely.</li>
-<li>An optional <a href="/docs/24.0.0/multi-stage-query/index.html#clustered-by">CLUSTERED BY</a> clause.</li>
+<li>A clause for the data you want to insert, such as <code>SELECT ... FROM ...</code>. You can use <a href="#extern">EXTERN</a> to reference external tables using <code>FROM TABLE(EXTERN(...))</code>.</li>
+<li>A <a href="#partitioned-by">PARTITIONED BY</a> clause, such as <code>PARTITIONED BY DAY</code>.</li>
+<li>An optional <a href="#clustered-by">CLUSTERED BY</a> clause.</li>
</ol>
+<p>For more information, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#insert">Load data with INSERT</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="replace"></a><a href="#replace" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
<p>You can use the REPLACE function to replace all or some of the data.</p>
-<p>Unlike standard SQL, REPLACE inserts data according to column name and not positionally. This means that it is important for the output column names of subsequent REPLACE queries to be the same as the table. Do not rely on their positions within the SELECT clause.</p>
+<p>Unlike standard SQL, REPLACE loads data into the target table according to column name, not positionally. If necessary,
+use <code>AS</code> in your SELECT column list to assign the correct names. Do not rely on their positions within the SELECT
+clause.</p>
<h4><a class="anchor" aria-hidden="true" id="replace-all-data"></a><a href="#replace-all-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
<p>Function format to replace all data:</p>
<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> <target <span class="hljs-keyword">table</span>>
OVERWRITE <span class="hljs-keyword">ALL</span>
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <<span class="hljs-keyword">column</span>>
-<span class="hljs-keyword">FROM</span> <<span class="hljs-keyword">source</span> <span class="hljs-keyword">table</span>>
-
-PARTITIONED <span class="hljs-keyword">BY</span> <<span class="hljs-built_in">time</span>>
+< <span class="hljs-keyword">SELECT</span> <span class="hljs-keyword">query</span> >
+PARTITIONED <span class="hljs-keyword">BY</span> <<span class="hljs-built_in">time</span> granularity>
+[ CLUSTERED <span class="hljs-keyword">BY</span> <<span class="hljs-keyword">column</span> <span class="hljs-keyword">list</span>> ]
</code></pre>
-<h4><a class="anchor" aria-hidden="true" id="replace-specific-data"></a><a href="#replace-specific-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
-<p>Function format to replace specific data:</p>
+<h4><a class="anchor" aria-hidden="true" id="replace-specific-time-ranges"></a><a href="#replace-specific-time-ranges" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
+<p>Function format to replace specific time ranges:</p>
<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> <target <span class="hljs-keyword">table</span>>
OVERWRITE <span class="hljs-keyword">WHERE</span> __time >= <span class="hljs-built_in">TIMESTAMP</span> <span class="hljs-string">'<lower bound>'</span> <span class="hljs-keyword">AND</span> __time < <span class="hljs-built_in">TIMESTAMP</span> <span class="hljs-string">'<upper bound>'</span>
-<span class="hljs-keyword">SELECT</span>
- TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
- <<span class="hljs-keyword">column</span>>
-<span class="hljs-keyword">FROM</span> <<span class="hljs-keyword">source</span> <span class="hljs-keyword">table</span>>
-
-PARTITIONED <span class="hljs-keyword">BY</span> <<span class="hljs-built_in">time</span>>
+< <span class="hljs-keyword">SELECT</span> <span class="hljs-keyword">query</span> >
+PARTITIONED <span class="hljs-keyword">BY</span> <<span class="hljs-built_in">time</span> granularity>
+[ CLUSTERED <span class="hljs-keyword">BY</span> <<span class="hljs-keyword">column</span> <span class="hljs-keyword">list</span>> ]
</code></pre>
<p>REPLACE consists of the following parts:</p>
<ol>
<li>Optional <a href="/docs/24.0.0/multi-stage-query/reference.html#context-parameters">context parameters</a>.</li>
-<li>A <code>REPLACE INTO <dataSource></code> clause at the start of your query, such as <code>REPLACE INTO your-table.</code></li>
+<li>A <code>REPLACE INTO <dataSource></code> clause at the start of your query, such as <code>REPLACE INTO "your-table".</code></li>
<li>An OVERWRITE clause after the datasource, either OVERWRITE ALL or OVERWRITE WHERE:</li>
</ol>
<ul>
<li>OVERWRITE ALL replaces the entire existing datasource with the results of the query.</li>
-<li>OVERWRITE WHERE drops the time segments that match the condition you set. Conditions are based on the <code>__time</code> column and use the format <code>__time [< > = <= >=] TIMESTAMP</code>. Use them with AND, OR, and NOT between them, inclusive of the timestamps specified. For example, see <a href="/docs/24.0.0/multi-stage-query/index.html#replace-some-data">REPLACE INTO ... OVERWRITE WHERE ... SELECT</a>.</li>
+<li>OVERWRITE WHERE drops the time segments that match the condition you set. Conditions are based on the <code>__time</code>
+column and use the format <code>__time [< > = <= >=] TIMESTAMP</code>. Use them with AND, OR, and NOT between them, inclusive
+of the timestamps specified. No other expressions or functions are valid in OVERWRITE.</li>
</ul>
<ol start="4">
<li>A clause for the actual data you want to use for the replacement.</li>
-<li>A <a href="/docs/24.0.0/multi-stage-query/index.html#partitioned-by">PARTITIONED BY</a> clause to your REPLACE statement. For example, use PARTITIONED BY DAY for daily partitioning, or PARTITIONED BY ALL TIME to skip time partitioning completely.</li>
-<li>An optional <a href="/docs/24.0.0/multi-stage-query/index.html#clustered-by">CLUSTERED BY</a> clause.</li>
+<li>A <a href="#partitioned-by">PARTITIONED BY</a> clause, such as <code>PARTITIONED BY DAY</code>.</li>
+<li>An optional <a href="#clustered-by">CLUSTERED BY</a> clause.</li>
</ol>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/security.html"><span class="arrow-prev">← </span><span>Security</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/known-issues.html"><span>Known issues</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#context-parameters">Context parameters</a></li><li><a href="#error-cod [...]
+<p>For more information, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#replace">Overwrite data with REPLACE</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="partitioned-by"></a><a href="#partitioned-by" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
+<p>The <code>PARTITIONED BY <time granularity></code> clause is required for <a href="#insert">INSERT</a> and <a href="#replace">REPLACE</a>. See
+<a href="/docs/24.0.0/multi-stage-query/concepts.html#partitioning">Partitioning</a> for details.</p>
+<p>The following granularity arguments are accepted:</p>
+<ul>
+<li>Time unit: <code>HOUR</code>, <code>DAY</code>, <code>MONTH</code>, or <code>YEAR</code>. Equivalent to <code>FLOOR(__time TO TimeUnit)</code>.</li>
+<li><code>TIME_FLOOR(__time, 'granularity_string')</code>, where granularity_string is one of the ISO 8601 periods listed below. The
+first argument must be <code>__time</code>.</li>
+<li><code>FLOOR(__time TO TimeUnit)</code>, where <code>TimeUnit</code> is any unit supported by the <a href="/docs/24.0.0/querying/sql-scalar.html#date-and-time-functions">FLOOR function</a>. The first argument must be <code>__time</code>.</li>
+<li><code>ALL</code> or <code>ALL TIME</code>, which effectively disables time partitioning by placing all data in a single time chunk. To use
+LIMIT or OFFSET at the outer level of your INSERT or REPLACE query, you must set PARTITIONED BY to ALL or ALL TIME.</li>
+</ul>
+<p>The following ISO 8601 periods are supported for <code>TIME_FLOOR</code>:</p>
+<ul>
+<li>PT1S</li>
+<li>PT1M</li>
+<li>PT5M</li>
+<li>PT10M</li>
+<li>PT15M</li>
+<li>PT30M</li>
+<li>PT1H</li>
+<li>PT6H</li>
+<li>P1D</li>
+<li>P1W</li>
+<li>P1M</li>
+<li>P3M</li>
+<li>P1Y</li>
+</ul>
+<p>For more information about partitioning, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#partitioning">Partitioning</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="clustered-by"></a><a href="#clustered-by" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>The <code>CLUSTERED BY <column list></code> clause is optional for <a href="#insert">INSERT</a> and <a href="#replace">REPLACE</a>. It accepts a list of
+column names or expressions.</p>
+<p>For more information about clustering, see <a href="/docs/24.0.0/multi-stage-query/concepts.html#clustering">Clustering</a>.</p>
+<p><a name="context"></a></p>
+<h2><a class="anchor" aria-hidden="true" id="context-parameters"></a><a href="#context-parameters" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
+<p>In addition to the Druid SQL <a href="/docs/24.0.0/querying/sql-query-context.html">context parameters</a>, the multi-stage query task engine accepts certain context parameters that are specific to it.</p>
+<p>Use context parameters alongside your queries to customize the behavior of the query. If you're using the API, include the context parameters in the query context when you submit a query:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"query"</span>: <span class="hljs-string">"SELECT 1 + 1"</span>,
+ <span class="hljs-attr">"context"</span>: {
+ <span class="hljs-attr">"<key>"</span>: <span class="hljs-string">"<value>"</span>,
+ <span class="hljs-attr">"maxNumTasks"</span>: <span class="hljs-number">3</span>
+ }
+}
+</code></pre>
+<p>If you're using the web console, you can specify the context parameters through various UI options.</p>
+<p>The following table lists the context parameters for the MSQ task engine:</p>
+<table>
+<thead>
+<tr><th>Parameter</th><th>Description</th><th>Default value</th></tr>
+</thead>
+<tbody>
+<tr><td>maxNumTasks</td><td>SELECT, INSERT, REPLACE<br /><br />The maximum total number of tasks to launch, including the controller task. The lowest possible value for this setting is 2: one controller and one worker. All tasks must be able to launch simultaneously. If they cannot, the query returns a <code>TaskStartTimeout</code> error code after approximately 10 minutes.<br /><br />May also be provided as <code>numTasks</code>. If both are present, <code>maxNumTasks</code> takes prior [...]
+<tr><td>taskAssignment</td><td>SELECT, INSERT, REPLACE<br /><br />Determines how many tasks to use. Possible values include: <ul><li><code>max</code>: Uses as many tasks as possible, up to <code>maxNumTasks</code>.</li><li><code>auto</code>: When file sizes can be determined through directory listing (for example: local files, S3, GCS, HDFS) uses as few tasks as possible without exceeding 10 GiB or 10,000 files per task, unless exceeding these limits is necessary to stay within <code>max [...]
+<tr><td>finalizeAggregations</td><td>SELECT, INSERT, REPLACE<br /><br />Determines the type of aggregation to return. If true, Druid finalizes the results of complex aggregations that directly appear in query results. If false, Druid returns the aggregation's intermediate type rather than finalized type. This parameter is useful during ingestion, where it enables storing sketches directly in Druid tables. For more information about aggregations, see <a href="/docs/24.0.0/querying/sql-agg [...]
+<tr><td>rowsInMemory</td><td>INSERT or REPLACE<br /><br />Maximum number of rows to store in memory at once before flushing to disk during the segment generation process. Ignored for non-INSERT queries. In most cases, use the default value. You may need to override the default if you run into one of the <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> around memory usage.</td><td>100,000</td></tr>
+<tr><td>segmentSortOrder</td><td>INSERT or REPLACE<br /><br />Normally, Druid sorts rows in individual segments using <code>__time</code> first, followed by the <a href="#clustered-by">CLUSTERED BY</a> clause. When you set <code>segmentSortOrder</code>, Druid sorts rows in segments using this column list first, followed by the CLUSTERED BY order.<br /><br />You provide the column list as comma-separated values or as a JSON array in string form. If your query includes <code>__time</code>, [...]
+<tr><td>maxParseExceptions</td><td>SELECT, INSERT, REPLACE<br /><br />Maximum number of parse exceptions that are ignored while executing the query before it stops with <code>TooManyWarningsFault</code>. To ignore all the parse exceptions, set the value to -1.</td><td>0</td></tr>
+<tr><td>rowsPerSegment</td><td>INSERT or REPLACE<br /><br />The number of rows per segment to target. The actual number of rows per segment may be somewhat higher or lower than this number. In most cases, use the default. For general information about sizing rows per segment, see <a href="/docs/24.0.0/operations/segment-optimization.html">Segment Size Optimization</a>.</td><td>3,000,000</td></tr>
+<tr><td>sqlTimeZone</td><td>Sets the time zone for this connection, which affects how time functions and timestamp literals behave. Use a time zone name like "America/Los_Angeles" or offset like "-08:00".</td><td><code>druid.sql.planner.sqlTimeZone</code> on the Broker (default: UTC)</td></tr>
+<tr><td>useApproximateCountDistinct</td><td>Whether to use an approximate cardinality algorithm for <code>COUNT(DISTINCT foo)</code>.</td><td><code>druid.sql.planner.useApproximateCountDistinct</code> on the Broker (default: true)</td></tr>
+</tbody>
+</table>
+<h2><a class="anchor" aria-hidden="true" id="limits"></a><a href="#limits" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
+<p>Knowing the limits for the MSQ task engine can help you troubleshoot any <a href="#error-codes">errors</a> that you encounter. Many of the errors occur as a result of reaching a limit.</p>
+<p>The following table lists query limits:</p>
+<table>
+<thead>
+<tr><th>Limit</th><th>Value</th><th>Error if exceeded</th></tr>
+</thead>
+<tbody>
+<tr><td>Size of an individual row written to a frame. Row size when written to a frame may differ from the original row size.</td><td>1 MB</td><td><code>RowTooLarge</code></td></tr>
+<tr><td>Number of segment-granular time chunks encountered during ingestion.</td><td>5,000</td><td><code>TooManyBuckets</code></td></tr>
+<tr><td>Number of input files/segments per worker.</td><td>10,000</td><td><code>TooManyInputFiles</code></td></tr>
+<tr><td>Number of output partitions for any one stage. Number of segments generated during ingestion.</td><td>25,000</td><td><code>TooManyPartitions</code></td></tr>
+<tr><td>Number of output columns for any one stage.</td><td>2,000</td><td><code>TooManyColumns</code></td></tr>
+<tr><td>Number of workers for any one stage.</td><td>Hard limit is 1,000. Memory-dependent soft limit may be lower.</td><td><code>TooManyWorkers</code></td></tr>
+<tr><td>Maximum memory occupied by broadcasted tables.</td><td>30% of each <a href="/docs/24.0.0/multi-stage-query/concepts.html#memory-usage">processor memory bundle</a>.</td><td><code>BroadcastTablesTooLarge</code></td></tr>
+</tbody>
+</table>
+<p><a name="errors"></a></p>
+<h2><a class="anchor" aria-hidden="true" id="error-codes"></a><a href="#error-codes" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>The following table describes error codes you may encounter in the <code>multiStageQuery.payload.status.errorReport.error.errorCode</code> field:</p>
+<table>
+<thead>
+<tr><th>Code</th><th>Meaning</th><th>Additional fields</th></tr>
+</thead>
+<tbody>
+<tr><td>BroadcastTablesTooLarge</td><td>The size of the broadcast tables, used in right hand side of the joins, exceeded the memory reserved for them in a worker task.</td><td><code>maxBroadcastTablesSize</code>: Memory reserved for the broadcast tables, measured in bytes.</td></tr>
+<tr><td>Canceled</td><td>The query was canceled. Common reasons for cancellation:<br /><br /><ul><li>User-initiated shutdown of the controller task via the <code>/druid/indexer/v1/task/{taskId}/shutdown</code> API.</li><li>Restart or failure of the server process that was running the controller task.</li></ul></td><td></td></tr>
+<tr><td>CannotParseExternalData</td><td>A worker task could not parse data from an external datasource.</td><td></td></tr>
+<tr><td>ColumnNameRestricted</td><td>The query uses a restricted column name.</td><td></td></tr>
+<tr><td>ColumnTypeNotSupported</td><td>Support for writing or reading from a particular column type is not supported.</td><td></td></tr>
+<tr><td>ColumnTypeNotSupported</td><td>The query attempted to use a column type that is not supported by the frame format. This occurs with ARRAY types, which are not yet implemented for frames.</td><td><code>columnName</code><br /> <br /><code>columnType</code></td></tr>
+<tr><td>InsertCannotAllocateSegment</td><td>The controller task could not allocate a new segment ID due to conflict with existing segments or pending segments. Common reasons for such conflicts:<br /> <br /><ul><li>Attempting to mix different granularities in the same intervals of the same datasource.</li><li>Prior ingestions that used non-extendable shard specs.</li></ul></td><td><code>dataSource</code><br /> <br /><code>interval</code>: The interval for the attempted new segment alloca [...]
+<tr><td>InsertCannotBeEmpty</td><td>An INSERT or REPLACE query did not generate any output rows in a situation where output rows are required for success. This can happen for INSERT or REPLACE queries with <code>PARTITIONED BY</code> set to something other than <code>ALL</code> or <code>ALL TIME</code>.</td><td><code>dataSource</code></td></tr>
+<tr><td>InsertCannotOrderByDescending</td><td>An INSERT query contained a <code>CLUSTERED BY</code> expression in descending order. Druid's segment generation code only supports ascending order.</td><td><code>columnName</code></td></tr>
+<tr><td>InsertCannotReplaceExistingSegment</td><td>A REPLACE query cannot proceed because an existing segment partially overlaps those bounds, and the portion within the bounds is not fully overshadowed by query results. <br /> <br />There are two ways to address this without modifying your query:<ul><li>Shrink the OVERLAP filter to match the query results.</li><li>Expand the OVERLAP filter to fully contain the existing segment.</li></ul></td><td><code>segmentId</code>: The existing segm [...]
+<tr><td>InsertLockPreempted</td><td>An INSERT or REPLACE query was canceled by a higher-priority ingestion job, such as a real-time ingestion task.</td><td></td></tr>
+<tr><td>InsertTimeNull</td><td>An INSERT or REPLACE query encountered a null timestamp in the <code>__time</code> field.<br /><br />This can happen due to using an expression like <code>TIME_PARSE(timestamp) AS __time</code> with a timestamp that cannot be parsed. (TIME_PARSE returns null when it cannot parse a timestamp.) In this case, try parsing your timestamps using a different function or pattern.<br /><br />If your timestamps may genuinely be null, consider using COALESCE to provid [...]
+<tr><td>InsertTimeOutOfBounds</td><td>A REPLACE query generated a timestamp outside the bounds of the TIMESTAMP parameter for your OVERWRITE WHERE clause.<br /> <br />To avoid this error, verify that the you specified is valid.</td><td><code>interval</code>: time chunk interval corresponding to the out-of-bounds timestamp</td></tr>
+<tr><td>InvalidNullByte</td><td>A string column included a null byte. Null bytes in strings are not permitted.</td><td><code>column</code>: The column that included the null byte</td></tr>
+<tr><td>QueryNotSupported</td><td>QueryKit could not translate the provided native query to a multi-stage query.<br /> <br />This can happen if the query uses features that aren't supported, like GROUPING SETS.</td><td></td></tr>
+<tr><td>RowTooLarge</td><td>The query tried to process a row that was too large to write to a single frame. See the <a href="#limits">Limits</a> table for the specific limit on frame size. Note that the effective maximum row size is smaller than the maximum frame size due to alignment considerations during frame writing.</td><td><code>maxFrameSize</code>: The limit on the frame size.</td></tr>
+<tr><td>TaskStartTimeout</td><td>Unable to launch all the worker tasks in time. <br /> <br />There might be insufficient available slots to start all the worker tasks simultaneously.<br /> <br /> Try splitting up the query into smaller chunks with lesser <code>maxNumTasks</code> number. Another option is to increase capacity.</td><td></td></tr>
+<tr><td>TooManyBuckets</td><td>Exceeded the number of partition buckets for a stage. Partition buckets are only used for <code>segmentGranularity</code> during INSERT queries. The most common reason for this error is that your <code>segmentGranularity</code> is too narrow relative to the data. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>maxBuckets</code>: The limit on buckets.</td></tr>
+<tr><td>TooManyInputFiles</td><td>Exceeded the number of input files/segments per worker. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>umInputFiles</code>: The total number of input files/segments for the stage.<br /><br /><code>maxInputFiles</code>: The maximum number of input files/segments per worker per stage.<br /><br /><code>minNumWorker</code>: The minimum number of workers required for a successful run.</td></tr>
+<tr><td>TooManyPartitions</td><td>Exceeded the number of partitions for a stage. The most common reason for this is that the final stage of an INSERT or REPLACE query generated too many segments. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>maxPartitions</code>: The limit on partitions which was exceeded</td></tr>
+<tr><td>TooManyColumns</td><td>Exceeded the number of columns for a stage. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>maxColumns</code>: The limit on columns which was exceeded.</td></tr>
+<tr><td>TooManyWarnings</td><td>Exceeded the allowed number of warnings of a particular type.</td><td><code>rootErrorCode</code>: The error code corresponding to the exception that exceeded the required limit. <br /><br /><code>maxWarnings</code>: Maximum number of warnings that are allowed for the corresponding <code>rootErrorCode</code>.</td></tr>
+<tr><td>TooManyWorkers</td><td>Exceeded the supported number of workers running simultaneously. See the <a href="#limits">Limits</a> table for the specific limit.</td><td><code>workers</code>: The number of simultaneously running workers that exceeded a hard or soft limit. This may be larger than the number of workers in any one stage if multiple stages are running simultaneously. <br /><br /><code>maxWorkers</code>: The hard or soft limit on workers that was exceeded.</td></tr>
+<tr><td>NotEnoughMemory</td><td>Insufficient memory to launch a stage.</td><td><code>serverMemory</code>: The amount of memory available to a single process.<br /><br /><code>serverWorkers</code>: The number of workers running in a single process.<br /><br /><code>serverThreads</code>: The number of threads in a single process.</td></tr>
+<tr><td>WorkerFailed</td><td>A worker task failed unexpectedly.</td><td><code>workerTaskId</code>: The ID of the worker task.</td></tr>
+<tr><td>WorkerRpcFailed</td><td>A remote procedure call to a worker task failed and could not recover.</td><td><code>workerTaskId</code>: the id of the worker task</td></tr>
+<tr><td>UnknownError</td><td>All other errors.</td><td></td></tr>
+</tbody>
+</table>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/examples.html"><span class="arrow-prev">← </span><span>Examples</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/known-issues.html"><span>Known issues</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#sql-reference">SQL reference</a><ul class="toc-headings"><li><a href=" [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/multi-stage-query/security.html b/docs/24.0.0/multi-stage-query/security.html
index 4327cac1..4f9a0126 100644
--- a/docs/24.0.0/multi-stage-query/security.html
+++ b/docs/24.0.0/multi-stage-query/security.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/msq-security.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion security</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/multi-stage-query/security.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">SQL-based ingestion security</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -78,24 +78,30 @@
~ under the License.
-->
<blockquote>
-<p>SQL-based ingestion using the multi-stage query task engine is our recommended solution starting in Druid 24.0. Alternative ingestion solutions, such as native batch and Hadoop-based ingestion systems, will still be supported. We recommend you read all <a href="/docs/24.0.0/multi-stage-query/known-issues.html">known issues</a> and test the feature in a development environment before rolling it out in production. Using the multi-stage query task engine with <code>SELECT</code> statemen [...]
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
</blockquote>
-<p>All authenticated users can use the multi-stage query task engine (MSQ task engine) through the UI and API if the extension is loaded. However, without additional permissions, users are not able to issue queries that read or write Druid datasources or external data. The permission you need depends on what you are trying to do.</p>
-<p>The permission required to submit a query depends on the type of query:</p>
+<p>All authenticated users can use the multi-stage query task engine (MSQ task engine) through the UI and API if the extension is loaded. However, without additional permissions, users are not able to issue queries that read or write Druid datasources or external data. The permission needed depends on what the user is trying to do.</p>
+<p>To submit a query:</p>
<ul>
-<li>SELECT from a Druid datasource requires the READ DATASOURCE permission on that
-datasource.</li>
-<li>INSERT or REPLACE into a Druid datasource requires the WRITE DATASOURCE permission on that
-datasource.</li>
-<li>EXTERN references to external data require READ permission on the resource name "EXTERNAL" of the resource type "EXTERNAL". Users without the correct permission encounter a 403 error when trying to run queries that include EXTERN.</li>
+<li>SELECT from a Druid datasource requires the READ DATASOURCE permission on that datasource.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/reference.html#insert">INSERT</a> or <a href="/docs/24.0.0/multi-stage-query/reference.html#replace">REPLACE</a> into a Druid datasource requires the WRITE DATASOURCE
+permission on that datasource.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/reference.html#extern">EXTERN</a> requires READ permission on a resource named "EXTERNAL" with type "EXTERNAL". Users without the correct
+permission encounter a 403 error when trying to run queries that include EXTERN.</li>
</ul>
-<p>Query tasks that you submit to the MSQ task engine are Overlord tasks, so they follow the Overlord's (indexer) model. This means that users with access to the Overlord API can perform some actions even if they didn't submit the query. The actions include retrieving the status or canceling a query. For more information about the Overlord API and the task API, see <a href="/docs/24.0.0/multi-stage-query/api.html">APIs for SQL-based ingestion</a>.</p>
-<p>To interact with a query through the Overlord API, you need the following permissions:</p>
+<p>Once a query is submitted, it executes as a <a href="/docs/24.0.0/multi-stage-query/concepts.html#execution-flow"><code>query_controller</code></a> task. Query tasks that
+users submit to the MSQ task engine are Overlord tasks, so they follow the Overlord's security model. This means that
+users with access to the Overlord API can perform some actions even if they didn't submit the query, including
+retrieving status or canceling a query. For more information about the Overlord API and the task API, see <a href="/docs/24.0.0/multi-stage-query/api.html">APIs for
+SQL-based ingestion</a>.</p>
+<p>To interact with a query through the Overlord API, users need the following permissions:</p>
<ul>
-<li>INSERT or REPLACE queries: You must have READ DATASOURCE permission on the output datasource.</li>
-<li>SELECT queries: You must have read permissions on the <code>__query_select</code> datasource, which is a stub datasource that gets created.</li>
+<li>INSERT or REPLACE queries: Users must have READ DATASOURCE permission on the output datasource.</li>
+<li>SELECT queries: Users must have read permissions on the <code>__query_select</code> datasource, which is a stub datasource that gets created.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/api.html"><span class="arrow-prev">← </span><span>API</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/reference.html"><span>Reference</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/ [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/api.html"><span class="arrow-prev">← </span><span>API</span></a><a class="docs-next button" href="/docs/24.0.0/multi-stage-query/examples.html"><span>Examples</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a href="/te [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/operations/alerts.html b/docs/24.0.0/operations/alerts.html
index a7dfb6c9..2da22b71 100644
--- a/docs/24.0.0/operations/alerts.html
+++ b/docs/24.0.0/operations/alerts.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/api-reference.html b/docs/24.0.0/operations/api-reference.html
index 05f1f448..641eb88b 100644
--- a/docs/24.0.0/operations/api-reference.html
+++ b/docs/24.0.0/operations/api-reference.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -482,7 +482,7 @@ to filter by interval and limit the number of results respectively.</p>
<li><code>/druid/coordinator/v1/compaction/progress?dataSource={dataSource}</code></li>
</ul>
<p>Returns the total size of segments awaiting compaction for the given dataSource.
-The specified dataSource must have <a href="/docs/24.0.0/ingestion/automatic-compaction.html">automatic compaction</a> enabled.</p>
+The specified dataSource must have <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a> enabled.</p>
<h5><a class="anchor" aria-hidden="true" id="get-10"></a><a href="#get-10" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
<ul>
<li><code>/druid/coordinator/v1/compaction/status</code></li>
@@ -531,7 +531,7 @@ will be set for them.</p>
<ul>
<li><code>/druid/coordinator/v1/config/compaction</code></li>
</ul>
-<p>Creates or updates the <a href="/docs/24.0.0/ingestion/automatic-compaction.html">automatic compaction</a> config for a dataSource.
+<p>Creates or updates the <a href="/docs/24.0.0/data-management/automatic-compaction.html">automatic compaction</a> config for a dataSource.
See <a href="/docs/24.0.0/configuration/index.html#automatic-compaction-dynamic-configuration">Automatic compaction dynamic configuration</a> for configuration details.</p>
<h5><a class="anchor" aria-hidden="true" id="delete-1"></a><a href="#delete-1" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
<ul>
@@ -626,6 +626,7 @@ only want the active leader to be considered in-service at the load balancer.</p
<li><code>/druid/indexer/v1/task/{taskId}/reports</code></li>
</ul>
<p>Retrieve a <a href="/docs/24.0.0/ingestion/tasks.html#task-reports">task completion report</a> for a task. Only works for completed tasks.</p>
+<p><a name="task-submit"></a></p>
<h5><a class="anchor" aria-hidden="true" id="post-5"></a><a href="#post-5" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2. [...]
<ul>
<li><code>/druid/indexer/v1/task</code></li>
diff --git a/docs/24.0.0/operations/auth-ldap.html b/docs/24.0.0/operations/auth-ldap.html
index 7d539eb5..2a302cde 100644
--- a/docs/24.0.0/operations/auth-ldap.html
+++ b/docs/24.0.0/operations/auth-ldap.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -148,7 +148,7 @@ druid.auth.authorizer.ldapauth.roleProvider.<span class="hljs-attribute">type</s
<p>The following command associates the permissions in the JSON file with the role</p>
<pre><code class="hljs">curl -i -v -H <span class="hljs-string">"Content-Type: application/json"</span> -u internal -X POST -d@perm.json http:<span class="hljs-regexp">//</span>localhost:<span class="hljs-number">8081</span><span class="hljs-regexp">/druid-ext/</span>basic-security<span class="hljs-regexp">/authorization/</span>db<span class="hljs-regexp">/ldapauth/</span>roles<span class="hljs-regexp">/readRole/</span>permissions
</code></pre>
-<p>Note that the STATE and CONFIG permissions in <code>perm.json</code> are needed to see the data source in the Druid console. If only querying permissions are needed, the READ action is sufficient:</p>
+<p>Note that the STATE and CONFIG permissions in <code>perm.json</code> are needed to see the data source in the web console. If only querying permissions are needed, the READ action is sufficient:</p>
<pre><code class="hljs">[{ <span class="hljs-attr">"resource"</span>: { <span class="hljs-attr">"name"</span>: <span class="hljs-string">"wikipedia"</span>, <span class="hljs-attr">"type"</span>: <span class="hljs-string">"DATASOURCE"</span> }, <span class="hljs-attr">"action"</span>: <span class="hljs-string">"READ"</span> }]
</code></pre>
<p>You can also provide the name in the form of regular expression. For example, to give access to all data sources starting with <code>wiki</code>, specify the name as <code>{ "name": "wiki.*", .....</code>.</p>
diff --git a/docs/24.0.0/operations/basic-cluster-tuning.html b/docs/24.0.0/operations/basic-cluster-tuning.html
index bee7b032..5bc1f79c 100644
--- a/docs/24.0.0/operations/basic-cluster-tuning.html
+++ b/docs/24.0.0/operations/basic-cluster-tuning.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -220,7 +220,7 @@
<h6><a class="anchor" aria-hidden="true" id="kafkakinesis-ingestion"></a><a href="#kafkakinesis-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
<p>If you use the <a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Kafka Indexing Service</a> or <a href="/docs/24.0.0/development/extensions-core/kinesis-ingestion.html">Kinesis Indexing Service</a>, the number of tasks required will depend on the number of partitions and your taskCount/replica settings.</p>
<p>On top of those requirements, allocating more task slots in your cluster is a good idea, so that you have free task
-slots available for other tasks, such as <a href="/docs/24.0.0/ingestion/compaction.html">compaction tasks</a>.</p>
+slots available for other tasks, such as <a href="/docs/24.0.0/data-management/compaction.html">compaction tasks</a>.</p>
<h6><a class="anchor" aria-hidden="true" id="hadoop-ingestion"></a><a href="#hadoop-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
<p>If you are only using <a href="/docs/24.0.0/ingestion/hadoop.html">Hadoop-based batch ingestion</a> with no other ingestion types, you can lower the amount of resources allocated per Task. Batch ingestion tasks do not need to answer queries, and the bulk of the ingestion workload will be executed on the Hadoop cluster, so the Tasks do not require much resources.</p>
<h6><a class="anchor" aria-hidden="true" id="parallel-native-ingestion"></a><a href="#parallel-native-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c [...]
diff --git a/docs/24.0.0/operations/clean-metadata-store.html b/docs/24.0.0/operations/clean-metadata-store.html
index 9c48df66..ac0f97e7 100644
--- a/docs/24.0.0/operations/clean-metadata-store.html
+++ b/docs/24.0.0/operations/clean-metadata-store.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/deep-storage-migration.html b/docs/24.0.0/operations/deep-storage-migration.html
index a23df23d..b9f81cd3 100644
--- a/docs/24.0.0/operations/deep-storage-migration.html
+++ b/docs/24.0.0/operations/deep-storage-migration.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/druid-console.html b/docs/24.0.0/operations/druid-console.html
index 01d68511..2e1407af 100644
--- a/docs/24.0.0/operations/druid-console.html
+++ b/docs/24.0.0/operations/druid-console.html
@@ -1,214 +1,8 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Druid console · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/operations/druid-console.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" />< [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/operations/druid-console.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Druid console</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<p>Druid includes a web console for loading data, managing datasources and tasks, and viewing server status and segment information.
-You can also run SQL and native Druid queries in the console.</p>
-<p>Enable the following cluster settings to use the Druid console. Note that these settings are enabled by default.</p>
-<ul>
-<li>Enable the Router's <a href="/docs/24.0.0/design/router.html#enabling-the-management-proxy">management proxy</a>.</li>
-<li>Enable <a href="/docs/24.0.0/configuration/index.html#sql">Druid SQL</a> for the Broker processes in the cluster.</li>
-</ul>
-<p>The <a href="/docs/24.0.0/design/router.html">Router</a> service hosts the Druid console.
-Access the Druid console at the following address:</p>
-<pre><code class="hljs">http://<span class="hljs-tag"><<span class="hljs-name">ROUTER_IP</span>></span>:<span class="hljs-tag"><<span class="hljs-name">ROUTER_PORT</span>></span>
-</code></pre>
-<blockquote>
-<p>It is important to note that any Druid console user will have, effectively, the same file permissions as the user under which Druid runs. One way these permissions are surfaced is in the file browser dialog. The dialog
-will show console users the files that the underlying user has permissions to. In general, avoid running Druid as
-root user. Consider creating a dedicated user account for running Druid.</p>
-</blockquote>
-<p>This topic presents the high-level features and functionality of the Druid console.</p>
-<h2><a class="anchor" aria-hidden="true" id="home"></a><a href="#home" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6 [...]
-<p>The <strong>Home</strong> view provides a high-level overview of the cluster.
-Each card is clickable and links to the appropriate view.</p>
-<p>The <strong>Home</strong> view displays the following cards:</p>
-<ul>
-<li><strong>Status</strong>. Click this card for information on the Druid version and any extensions loaded on the cluster.</li>
-<li><a href="#datasources">Datasources</a></li>
-<li><a href="#segments">Segments</a></li>
-<li><a href="#supervisors-and-tasks">Supervisors</a></li>
-<li><a href="#supervisors-and-tasks">Tasks</a></li>
-<li><a href="#services">Services</a></li>
-<li><a href="#lookups">Lookups</a></li>
-</ul>
-<p>You can access the <a href="#data-loader">data loader</a> and <a href="#lookups">lookups view</a> from the top-level navigation of the <strong>Home</strong> view.</p>
-<p><img src="../assets/web-console-01-home-view.png" alt="home-view" title="home view"></p>
-<h2><a class="anchor" aria-hidden="true" id="query"></a><a href="#query" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09 [...]
-<p>SQL-based ingestion and the multi-stage query task engine use the <strong>Query</strong> view, which provides you with a UI to edit and use SQL queries. You should see this UI automatically in Druid 24.0 and later since the multi-stage query extension is loaded by default.</p>
-<p>The following screenshot shows a populated enhanced <strong>Query</strong> view along with a description of its parts:</p>
-<p><img src="../assets/multi-stage-query/ui-annotated.png" alt="Annotated multi-stage Query view"></p>
-<ol>
-<li>The multi-stage, tab-enabled, <strong>Query</strong> view is where you can issue queries and see results.
-All other views are unchanged from the non-enhanced version. You can still access the original <strong>Query</strong> view by navigating to <code>#query</code> in the URL.
-You can tell that you're looking at the updated <strong>Query</strong> view by the presence of the tabs (3).</li>
-<li>The <strong>druid</strong> panel shows the available schemas, datasources, and columns.</li>
-<li>Query tabs allow you to manage and run several queries at once.
-Click the plus icon to open a new tab.
-To manipulate existing tabs, click the tab name.</li>
-<li>The tab bar contains some helpful tools including the <strong>Connect external data</strong> button that samples external data and creates an initial query with the appropriate <code>EXTERN</code> definition that you can then edit as needed.</li>
-<li>The <strong>Recent query tasks</strong> panel lets you see currently running and previous queries from all users in the cluster.
-It is equivalent to the <strong>Task</strong> view in the <strong>Ingestion</strong> view with the filter of <code>type='query_controller'</code>.</li>
-<li>You can click on each query entry to attach to that query in a new tab.</li>
-<li>You can download an archive of all the pertinent details about the query that you can share.</li>
-<li>The <strong>Run</strong> button runs the query.</li>
-<li>The <strong>Preview</strong> button appears when you enter an INSERT/REPLACE query. It runs the query inline without the INSERT/REPLACE clause and with an added LIMIT to give you a preview of the data that would be ingested if you click <strong>Run</strong>.
-The added LIMIT makes the query run faster but provides incomplete results.</li>
-<li>The engine selector lets you choose which engine (API endpoint) to send a query to. By default, it automatically picks which endpoint to use based on an analysis of the query, but you can select a specific engine explicitly. You can also configure the engine specific context parameters from this menu.</li>
-<li>The <strong>Max tasks</strong> picker appears when you have the <strong>sql-msq-task</strong> engine selected. It lets you configure the degree of parallelism.</li>
-<li>The More menu (<strong>...</strong>) contains the following helpful tools:</li>
-</ol>
-<ul>
-<li><strong>Explain SQL query</strong> shows you the logical plan returned by <code>EXPLAIN PLAN FOR</code> for a SQL query.</li>
-<li><strong>Query history</strong> shows you previously executed queries.</li>
-<li><strong>Convert ingestion spec to SQL</strong> lets you convert a native batch ingestion spec to an equivalent SQL query.</li>
-<li><strong>Attach tab from task ID</strong> lets you create a new tab from the task ID of a query executed on this cluster.</li>
-<li><strong>Open query detail archive</strong> lets you open a detail archive generated on any cluster by (7).</li>
-</ul>
-<ol start="13">
-<li>The query timer indicates how long the query has been running for.</li>
-<li>The <strong>(cancel)</strong> link cancels the currently running query.</li>
-<li>The main progress bar shows the overall progress of the query.
-The progress is computed from the various counters in the live reports (16).</li>
-<li>The <strong>Current stage</strong> progress bar shows the progress for the currently running query stage.
-If several stages are executing concurrently, it conservatively shows the information for the earliest executing stage.</li>
-<li>The live query reports show detailed information of all the stages (past, present, and future). The live reports are shown while the query is running. You can hide the report if you want.
-After queries finish, you can access them by clicking on the query time indicator or from the <strong>Recent query tasks</strong> panel (6).</li>
-<li>You can expand each stage of the live query report by clicking on the triangle to show per worker and per partition statistics.</li>
-</ol>
-<h2><a class="anchor" aria-hidden="true" id="data-loader"></a><a href="#data-loader" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>You can use the data loader to build an ingestion spec with a step-by-step wizard.</p>
-<p><img src="../assets/web-console-02-data-loader-1.png" alt="data-loader-1"></p>
-<p>After selecting the location of your data, follow the series of steps displaying incremental previews of the data as it is ingested.
-After filling in the required details on every step you can navigate to the next step by clicking <strong>Next</strong>.
-You can also freely navigate between the steps from the top navigation.</p>
-<p>Navigating with the top navigation leaves the underlying spec unmodified while clicking <strong>Next</strong> attempts to fill in the subsequent steps with appropriate defaults.</p>
-<p><img src="../assets/web-console-03-data-loader-2.png" alt="data-loader-2"></p>
-<h2><a class="anchor" aria-hidden="true" id="datasources"></a><a href="#datasources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
-<p>The <strong>Datasources</strong> view shows all the datasources currently loaded on the cluster, as well as their sizes and availability.
-From the <strong>Datasources</strong> view, you can edit the retention rules, configure automatic compaction, and drop data in a datasource.</p>
-<p>A datasource is partitioned into one or more segments organized by time chunks.
-To display a timeline of segments, toggle the option for <strong>Show segment timeline</strong>.</p>
-<p>Like any view that is powered by a Druid SQL query, you can click <strong>View SQL query for table</strong> from the ellipsis menu to run the underlying SQL query directly.</p>
-<p><img src="../assets/web-console-04-datasources.png" alt="datasources"></p>
-<p>You can view and edit retention rules to determine the general availability of a datasource.</p>
-<p><img src="../assets/web-console-05-retention.png" alt="retention"></p>
-<h2><a class="anchor" aria-hidden="true" id="segments"></a><a href="#segments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
-<p>The <strong>Segments</strong> view shows all the <a href="/docs/24.0.0/design/segments.html">segments</a> in the cluster.
-Each segment has a detail view that provides more information.
-The Segment ID is also conveniently broken down into Datasource, Start, End, Version, and Partition columns for ease of filtering and sorting.</p>
-<p><img src="../assets/web-console-06-segments.png" alt="segments"></p>
-<h2><a class="anchor" aria-hidden="true" id="supervisors-and-tasks"></a><a href="#supervisors-and-tasks" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
-<p>From this view, you can check the status of existing supervisors as well as suspend, resume, and reset them.
-The supervisor oversees the state of the indexing tasks to coordinate handoffs, manage failures, and ensure that the scalability and replication requirements are maintained.</p>
-<p>The tasks table allows you to see the currently running and recently completed tasks.
-To navigate your tasks more easily, you can group them by their <strong>Type</strong>, <strong>Datasource</strong>, or <strong>Status</strong>.
-Submit a task manually by clicking the ellipsis icon and selecting <strong>Submit JSON task</strong>.</p>
-<p><img src="../assets/web-console-07-supervisors.png" alt="supervisors"></p>
-<p>Click on the magnifying glass for any supervisor to see detailed reports of its progress.</p>
-<p><img src="../assets/web-console-08-supervisor-status.png" alt="supervisor-status"></p>
-<p>Click on the magnifying glass for any task to see more detail about it.</p>
-<p><img src="../assets/web-console-09-task-status.png" alt="tasks-status"></p>
-<h2><a class="anchor" aria-hidden="true" id="services"></a><a href="#services" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
-<p>The <strong>Services</strong> view lets you see the current status of the nodes making up your cluster.
-You can group the nodes by type or by tier to get meaningful summary statistics.</p>
-<p><img src="../assets/web-console-10-servers.png" alt="servers"></p>
-<h2><a class="anchor" aria-hidden="true" id="lookups"></a><a href="#lookups" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
-<p>Access the <strong>Lookups</strong> view from the <strong>Lookups</strong> card in the home view or by clicking on the gear icon in the upper right corner.
-Here you can create and edit query time <a href="/docs/24.0.0/querying/lookups.html">lookups</a>.</p>
-<p><img src="../assets/web-console-13-lookups.png" alt="lookups"></p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/configuration/logging.html"><span class="arrow-prev">← </span><span>Logging</span></a><a class="docs-next button" href="/docs/24.0.0/operations/getting-started.html"><span>Getting started with Apache Druid</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#home">Home</a></li><li><a href="#query">Query</a></li><li>< [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="web-console.html">
+<meta http-equiv="refresh" content="0; url=web-console.html">
+<h1>Redirecting...</h1>
+<a href="web-console.html">Click here if you are not redirected.</a>
+<script>location="web-console.html"</script>
diff --git a/docs/24.0.0/operations/dump-segment.html b/docs/24.0.0/operations/dump-segment.html
index 60acd450..b1126822 100644
--- a/docs/24.0.0/operations/dump-segment.html
+++ b/docs/24.0.0/operations/dump-segment.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/dynamic-config-provider.html b/docs/24.0.0/operations/dynamic-config-provider.html
index 4d335087..6fb4387e 100644
--- a/docs/24.0.0/operations/dynamic-config-provider.html
+++ b/docs/24.0.0/operations/dynamic-config-provider.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/export-metadata.html b/docs/24.0.0/operations/export-metadata.html
index 6661038a..66ed2f7b 100644
--- a/docs/24.0.0/operations/export-metadata.html
+++ b/docs/24.0.0/operations/export-metadata.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/getting-started.html b/docs/24.0.0/operations/getting-started.html
index 156fdb38..29e4c616 100644
--- a/docs/24.0.0/operations/getting-started.html
+++ b/docs/24.0.0/operations/getting-started.html
@@ -3,63 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/operations/getting-started.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Getting started with Apache Druid</h1></header><article><div><span><!--
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -89,7 +33,7 @@
<p>The <a href="/docs/24.0.0/operations/basic-cluster-tuning.html">basic cluster tuning guide</a> is an introductory guide for tuning your Druid cluster.</p>
<h2><a class="anchor" aria-hidden="true" id="need-help-with-druid"></a><a href="#need-help-with-druid" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1 [...]
<p>If you have questions about using Druid, please reach out to the <a href="https://druid.apache.org/community/">Druid user mailing list or other community channels</a>!</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/druid-console.html"><span class="arrow-prev">← </span><span>Druid console</span></a><a class="docs-next button" href="/docs/24.0.0/operations/java.html"><span>Java runtime</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#overview">Overview</a></li><li><a href="#single-server-quickstart-and-tutorials">S [...]
+</span></div></article></div><div class="docs-prevnext"></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#overview">Overview</a></li><li><a href="#single-server-quickstart-and-tutorials">Single-server Quickstart and Tutorials</a></li><li><a href="#deploying-a-druid-cluster">Deploying a Druid cluster</a></li><li><a href="#operating-druid">Operating Druid</a></li><li><a href="#need-help-with-druid">Need help with Druid?</a></li></ul></nav></div><footer class=" [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/operations/high-availability.html b/docs/24.0.0/operations/high-availability.html
index 61bf36b6..6112d350 100644
--- a/docs/24.0.0/operations/high-availability.html
+++ b/docs/24.0.0/operations/high-availability.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/http-compression.html b/docs/24.0.0/operations/http-compression.html
index 967e25b1..e2641697 100644
--- a/docs/24.0.0/operations/http-compression.html
+++ b/docs/24.0.0/operations/http-compression.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/insert-segment-to-db.html b/docs/24.0.0/operations/insert-segment-to-db.html
index 0be91d30..8e452e30 100644
--- a/docs/24.0.0/operations/insert-segment-to-db.html
+++ b/docs/24.0.0/operations/insert-segment-to-db.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/java.html b/docs/24.0.0/operations/java.html
index 348d044b..7bb15939 100644
--- a/docs/24.0.0/operations/java.html
+++ b/docs/24.0.0/operations/java.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -128,7 +128,7 @@ a line like the following:</p>
</code></pre>
<p>The <code>Xms</code>, <code>Xmx</code>, and <code>MaxDirectMemorySize</code> parameters in the line above are merely an example. You may use different
values in your specific environment.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/getting-started.html"><span class="arrow-prev">← </span><span>Getting started with Apache Druid</span></a><a class="docs-next button" href="/docs/24.0.0/operations/security-overview.html"><span>Security overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#selecting-a-java-runtime">Selecting a Java [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/web-console.html"><span class="arrow-prev">← </span><span>Web console</span></a><a class="docs-next button" href="/docs/24.0.0/operations/security-overview.html"><span>Security overview</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#selecting-a-java-runtime">Selecting a Java runtime</a></li><li><a hr [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/operations/kubernetes.html b/docs/24.0.0/operations/kubernetes.html
index 335f1d99..666f850f 100644
--- a/docs/24.0.0/operations/kubernetes.html
+++ b/docs/24.0.0/operations/kubernetes.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/management-uis.html b/docs/24.0.0/operations/management-uis.html
new file mode 100644
index 00000000..2e1407af
--- /dev/null
+++ b/docs/24.0.0/operations/management-uis.html
@@ -0,0 +1,8 @@
+<!DOCTYPE html>
+<meta charset="utf-8">
+<title>Redirecting...</title>
+<link rel="canonical" href="web-console.html">
+<meta http-equiv="refresh" content="0; url=web-console.html">
+<h1>Redirecting...</h1>
+<a href="web-console.html">Click here if you are not redirected.</a>
+<script>location="web-console.html"</script>
diff --git a/docs/24.0.0/operations/metadata-migration.html b/docs/24.0.0/operations/metadata-migration.html
index 3d9885f9..ecc2c081 100644
--- a/docs/24.0.0/operations/metadata-migration.html
+++ b/docs/24.0.0/operations/metadata-migration.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/metrics.html b/docs/24.0.0/operations/metrics.html
index dfbcf3f1..874e255a 100644
--- a/docs/24.0.0/operations/metrics.html
+++ b/docs/24.0.0/operations/metrics.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/mixed-workloads.html b/docs/24.0.0/operations/mixed-workloads.html
index ba43ef33..8c4052ec 100644
--- a/docs/24.0.0/operations/mixed-workloads.html
+++ b/docs/24.0.0/operations/mixed-workloads.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/other-hadoop.html b/docs/24.0.0/operations/other-hadoop.html
index 80b624f1..1275d8d3 100644
--- a/docs/24.0.0/operations/other-hadoop.html
+++ b/docs/24.0.0/operations/other-hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/password-provider.html b/docs/24.0.0/operations/password-provider.html
index af01aa90..c2f1ffe0 100644
--- a/docs/24.0.0/operations/password-provider.html
+++ b/docs/24.0.0/operations/password-provider.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/pull-deps.html b/docs/24.0.0/operations/pull-deps.html
index 2c1e5de2..e8234749 100644
--- a/docs/24.0.0/operations/pull-deps.html
+++ b/docs/24.0.0/operations/pull-deps.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/request-logging.html b/docs/24.0.0/operations/request-logging.html
index 6ec0d2f4..c5dd0269 100644
--- a/docs/24.0.0/operations/request-logging.html
+++ b/docs/24.0.0/operations/request-logging.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/reset-cluster.html b/docs/24.0.0/operations/reset-cluster.html
index 3be5ddce..670d29a1 100644
--- a/docs/24.0.0/operations/reset-cluster.html
+++ b/docs/24.0.0/operations/reset-cluster.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/rolling-updates.html b/docs/24.0.0/operations/rolling-updates.html
index 6439c27b..46213102 100644
--- a/docs/24.0.0/operations/rolling-updates.html
+++ b/docs/24.0.0/operations/rolling-updates.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/rule-configuration.html b/docs/24.0.0/operations/rule-configuration.html
index 38224358..bd35f215 100644
--- a/docs/24.0.0/operations/rule-configuration.html
+++ b/docs/24.0.0/operations/rule-configuration.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -77,7 +77,7 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<p>In Apache Druid, Coordinator processes use rules to determine what data should be loaded to or dropped from the cluster. Rules are used for data retention and query execution, and are set via the <a href="/docs/24.0.0/operations/druid-console.html">web console</a>.</p>
+<p>In Apache Druid, Coordinator processes use rules to determine what data should be loaded to or dropped from the cluster. Rules are used for data retention and query execution, and are set via the <a href="/docs/24.0.0/operations/web-console.html">web console</a>.</p>
<p>There are three types of rules, i.e., load rules, drop rules, and broadcast rules. Load rules indicate how segments should be assigned to different historical process tiers and how many replicas of a segment should exist in each tier.
Drop rules indicate when segments should be dropped entirely from the cluster. Finally, broadcast rules indicate how segments of different datasources should be co-located in Historical processes.</p>
<p>The Coordinator loads a set of rules from the metadata storage. Rules may be specific to a certain datasource and/or a
diff --git a/docs/24.0.0/operations/security-overview.html b/docs/24.0.0/operations/security-overview.html
index f82f14e4..1c7f8fb7 100644
--- a/docs/24.0.0/operations/security-overview.html
+++ b/docs/24.0.0/operations/security-overview.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -88,7 +88,7 @@
Druid administrators have the same OS permissions as the Unix user account running Druid. See <a href="/docs/24.0.0/operations/security-user-auth.html#authentication-and-authorization-model">Authentication and authorization model</a>. If the Druid process is running under the OS root user account, then Druid administrators can read or write all files that the root account has access to, including sensitive files such as <code>/etc/passwd</code>.</p>
</blockquote></li>
<li>Enable authentication to the Druid cluster for production environments and other environments that can be accessed by untrusted networks.</li>
-<li>Enable authorization and do not expose the Druid console without authorization enabled. If authorization is not enabled, any user that has access to the web console has the same privileges as the operating system user that runs the Druid console process.</li>
+<li>Enable authorization and do not expose the web console without authorization enabled. If authorization is not enabled, any user that has access to the web console has the same privileges as the operating system user that runs the web console process.</li>
<li>Grant users the minimum permissions necessary to perform their functions. For instance, do not allow users who only need to query data to write to data sources or view state.</li>
<li>Do not provide plain-text passwords for production systems in configuration specs. For example, sensitive properties should not be in the <code>consumerProperties</code> field of <code>KafkaSupervisorIngestionSpec</code>. See <a href="/docs/24.0.0/operations/dynamic-config-provider.html#environment-variable-dynamic-config-provider">Environment variable dynamic config provider</a> for more information.</li>
<li>Disable JavaScript, as noted in the <a href="https://druid.apache.org/docs/latest/development/javascript.html#security">Security section</a> of the JavaScript guide.</li>
@@ -106,7 +106,7 @@ Druid administrators have the same OS permissions as the Unix user account runni
</ul>
<p>The following recommendation applies to Druid's authorization and authentication model:</p>
<ul>
-<li>Only grant <code>WRITE</code> permissions to any <code>DATASOURCE</code> to trusted users. Druid's trust model assumes those users have the same privileges as the operating system user that runs the Druid console process. Additionally, users with <code>WRITE</code> permissions can make changes to datasources and they have access to both task and supervisor update (POST) APIs which may affect ingestion.</li>
+<li>Only grant <code>WRITE</code> permissions to any <code>DATASOURCE</code> to trusted users. Druid's trust model assumes those users have the same privileges as the operating system user that runs the web console process. Additionally, users with <code>WRITE</code> permissions can make changes to datasources and they have access to both task and supervisor update (POST) APIs which may affect ingestion.</li>
<li>Only grant <code>STATE READ</code>, <code>STATE WRITE</code>, <code>CONFIG WRITE</code>, and <code>DATASOURCE WRITE</code> permissions to highly-trusted users. These permissions allow users to access resources on behalf of the Druid server process regardless of the datasource.</li>
<li>If your Druid client application allows less-trusted users to control the input source or firehose of an ingestion task, validate the URLs from the users. It is possible to point unchecked URLs to other locations and resources within your network or local file system.</li>
</ul>
diff --git a/docs/24.0.0/operations/security-user-auth.html b/docs/24.0.0/operations/security-user-auth.html
index 5e3347cb..5c4eb078 100644
--- a/docs/24.0.0/operations/security-user-auth.html
+++ b/docs/24.0.0/operations/security-user-auth.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -85,7 +85,7 @@
<ul>
<li>DATASOURCE – Each Druid table (i.e., <code>tables</code> in the <code>druid</code> schema in SQL) is a resource.</li>
<li>CONFIG – Configuration resources exposed by the cluster components.</li>
-<li>EXTERNAL – External data read through the <a href="/docs/24.0.0/multi-stage-query/index.html#read-external-data">EXTERN function</a> in SQL.</li>
+<li>EXTERNAL – External data read through the <a href="/docs/24.0.0/multi-stage-query/concepts.html#extern">EXTERN function</a> in SQL.</li>
<li>STATE – Cluster-wide state resources.</li>
<li>SYSTEM_TABLE – when the Broker property <code>druid.sql.planner.authorizeSystemTablesDirectly</code> is true, then Druid uses this resource type to authorize the system tables in the <code>sys</code> schema in SQL.</li>
</ul>
@@ -145,7 +145,7 @@ This section describes the resource names available for each resource type.</p>
<h3><a class="anchor" aria-hidden="true" id="external"></a><a href="#external" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
<p>The EXTERNAL resource type only accepts the resource name "EXTERNAL".
Granting a user access to EXTERNAL resources allows them to run queries that include
-the <a href="/docs/24.0.0/multi-stage-query/index.html#read-external-data">EXTERN function</a> in SQL
+the <a href="/docs/24.0.0/multi-stage-query/concepts.html#extern">EXTERN function</a> in SQL
to read external data.</p>
<h3><a class="anchor" aria-hidden="true" id="state"></a><a href="#state" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09 [...]
<p>There is only one possible resource name for the "STATE" config resource type, "STATE". Granting a user access to STATE resources allows them to access the following endpoints.</p>
@@ -185,7 +185,7 @@ to read external data.</p>
<p><code>GET</code> requests require READ permissions, while <code>POST</code> and <code>DELETE</code> requests require WRITE permissions.</p>
<h3><a class="anchor" aria-hidden="true" id="sql-permissions"></a><a href="#sql-permissions" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
<p>Queries on Druid datasources require DATASOURCE READ permissions for the specified datasource.</p>
-<p>Queries to access external data through the <a href="/docs/24.0.0/multi-stage-query/index.html#read-external-data">EXTERN function</a> require EXTERNAL READ permissions.</p>
+<p>Queries to access external data through the <a href="/docs/24.0.0/multi-stage-query/concepts.html#extern">EXTERN function</a> require EXTERNAL READ permissions.</p>
<p>Queries on <a href="/docs/24.0.0/querying/sql-metadata-tables.html#information-schema">INFORMATION_SCHEMA tables</a> return information about datasources that the caller has DATASOURCE READ access to. Other
datasources are omitted.</p>
<p>Queries on the <a href="/docs/24.0.0/querying/sql-metadata-tables.html#system-schema">system schema tables</a> require the following permissions:</p>
diff --git a/docs/24.0.0/operations/segment-optimization.html b/docs/24.0.0/operations/segment-optimization.html
index a006248a..84d18e29 100644
--- a/docs/24.0.0/operations/segment-optimization.html
+++ b/docs/24.0.0/operations/segment-optimization.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -92,7 +92,7 @@ slots of the processing pool.</li>
</ol>
<p>It would be best if you can optimize the segment size at ingestion time, but sometimes it's not easy
especially when it comes to stream ingestion because the amount of data ingested might vary over time. In this case,
-you can create segments with a sub-optimized size first and optimize them later using <a href="/docs/24.0.0/ingestion/compaction.html">compaction</a>.</p>
+you can create segments with a sub-optimized size first and optimize them later using <a href="/docs/24.0.0/data-management/compaction.html">compaction</a>.</p>
<p>You may need to consider the followings to optimize your segments.</p>
<ul>
<li>Number of rows per segment: it's generally recommended for each segment to have around 5 million rows.
@@ -140,16 +140,16 @@ In this case, you may want to see only rows of the max version per interval (pai
<li>Turning on the <a href="/docs/24.0.0/design/coordinator.html#automatic-compaction">automatic compaction of Coordinators</a>.
The Coordinator periodically submits <a href="/docs/24.0.0/ingestion/tasks.html#compact">compaction tasks</a> to re-index small segments.
To enable the automatic compaction, you need to configure it for each dataSource via Coordinator's dynamic configuration.
-For more information, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a>.</li>
+For more information, see <a href="/docs/24.0.0/data-management/automatic-compaction.html">Automatic compaction</a>.</li>
<li>Running periodic Hadoop batch ingestion jobs and using a <code>dataSource</code>
inputSpec to read from the segments generated by the Kafka indexing tasks. This might be helpful if you want to compact a lot of segments in parallel.
-Details on how to do this can be found on the <a href="/docs/24.0.0/ingestion/data-management.html#update">Updating existing data</a> section
+Details on how to do this can be found on the <a href="/docs/24.0.0/data-management/update.html">Updating existing data</a> section
of the data management page.</li>
</ul>
<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<ul>
-<li>For an overview of compaction and how to submit a manual compaction task, see <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a>.</li>
-<li>To learn how to enable and configure automatic compaction, see <a href="/docs/24.0.0/ingestion/automatic-compaction.html">Automatic compaction</a>.</li>
+<li>For an overview of compaction and how to submit a manual compaction task, see <a href="/docs/24.0.0/data-management/compaction.html">Compaction</a>.</li>
+<li>To learn how to enable and configure automatic compaction, see <a href="/docs/24.0.0/data-management/automatic-compaction.html">Automatic compaction</a>.</li>
</ul>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/basic-cluster-tuning.html"><span class="arrow-prev">← </span><span>Basic cluster tuning</span></a><a class="docs-next button" href="/docs/24.0.0/operations/mixed-workloads.html"><span>Mixed workloads</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#learn-more">Learn more</a></li></ul></nav></div><foote [...]
document.addEventListener('keyup', function(e) {
diff --git a/docs/24.0.0/operations/single-server.html b/docs/24.0.0/operations/single-server.html
index 83dae822..375eee29 100644
--- a/docs/24.0.0/operations/single-server.html
+++ b/docs/24.0.0/operations/single-server.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -123,7 +123,7 @@
<li>Launch command: <code>bin/start-xlarge</code></li>
<li>Configuration directory: <code>conf/druid/single-server/xlarge</code></li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/docker.html"><span class="arrow-prev">← </span><span>Docker</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/cluster.html"><span>Clustered deployment</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#single-server-reference-configurations">Single server reference configurations</a><ul c [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/index.html"><span class="arrow-prev">← </span><span>Quickstart (local)</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/cluster.html"><span>Clustered deployment</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#single-server-reference-configurations">Single server reference configuratio [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/operations/tls-support.html b/docs/24.0.0/operations/tls-support.html
index bc7ea72d..a2a30d2d 100644
--- a/docs/24.0.0/operations/tls-support.html
+++ b/docs/24.0.0/operations/tls-support.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/use_sbt_to_build_fat_jar.html b/docs/24.0.0/operations/use_sbt_to_build_fat_jar.html
index 8c31a961..c08a4a77 100644
--- a/docs/24.0.0/operations/use_sbt_to_build_fat_jar.html
+++ b/docs/24.0.0/operations/use_sbt_to_build_fat_jar.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/operations/web-console.html b/docs/24.0.0/operations/web-console.html
new file mode 100644
index 00000000..6cd329cd
--- /dev/null
+++ b/docs/24.0.0/operations/web-console.html
@@ -0,0 +1,215 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Web console · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/operations/web-console.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><meta [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/operations/web-console.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Web console</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<p>Druid includes a web console for loading data, managing datasources and tasks, and viewing server status and segment information.
+You can also run SQL and native Druid queries in the console.</p>
+<p>Enable the following cluster settings to use the web console. Note that these settings are enabled by default.</p>
+<ul>
+<li>Enable the Router's <a href="/docs/24.0.0/design/router.html#enabling-the-management-proxy">management proxy</a>.</li>
+<li>Enable <a href="/docs/24.0.0/configuration/index.html#sql">Druid SQL</a> for the Broker processes in the cluster.</li>
+</ul>
+<p>The <a href="/docs/24.0.0/design/router.html">Router</a> service hosts the web console.
+Access the web console at the following address:</p>
+<pre><code class="hljs">http://<span class="hljs-tag"><<span class="hljs-name">ROUTER_IP</span>></span>:<span class="hljs-tag"><<span class="hljs-name">ROUTER_PORT</span>></span>
+</code></pre>
+<blockquote>
+<p><strong>Security note:</strong> Without <a href="/docs/24.0.0/operations/security-overview.html">Druid user permissions</a> configured, any user of the
+API or web console has effectively the same level of access to local files and network services as the user under which
+Druid runs. It is a best practice to avoid running Druid as the root user, and to use Druid permissions or network
+firewalls to restrict which users have access to potentially sensitive resources.</p>
+</blockquote>
+<p>This topic presents the high-level features and functionality of the web console.</p>
+<h2><a class="anchor" aria-hidden="true" id="home"></a><a href="#home" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09V6 [...]
+<p>The <strong>Home</strong> view provides a high-level overview of the cluster.
+Each card is clickable and links to the appropriate view.</p>
+<p>The <strong>Home</strong> view displays the following cards:</p>
+<ul>
+<li><strong>Status</strong>. Click this card for information on the Druid version and any extensions loaded on the cluster.</li>
+<li><a href="#datasources">Datasources</a></li>
+<li><a href="#segments">Segments</a></li>
+<li><a href="#supervisors-and-tasks">Supervisors</a></li>
+<li><a href="#supervisors-and-tasks">Tasks</a></li>
+<li><a href="#services">Services</a></li>
+<li><a href="#lookups">Lookups</a></li>
+</ul>
+<p>You can access the <a href="#data-loader">data loader</a> and <a href="#lookups">lookups view</a> from the top-level navigation of the <strong>Home</strong> view.</p>
+<p><img src="../assets/web-console-01-home-view.png" alt="home-view" title="home view"></p>
+<h2><a class="anchor" aria-hidden="true" id="query"></a><a href="#query" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1-2.09 [...]
+<p>SQL-based ingestion and the multi-stage query task engine use the <strong>Query</strong> view, which provides you with a UI to edit and use SQL queries. You should see this UI automatically in Druid 24.0 and later since the multi-stage query extension is loaded by default.</p>
+<p>The following screenshot shows a populated enhanced <strong>Query</strong> view along with a description of its parts:</p>
+<p><img src="../assets/multi-stage-query/ui-annotated.png" alt="Annotated multi-stage Query view"></p>
+<ol>
+<li>The multi-stage, tab-enabled, <strong>Query</strong> view is where you can issue queries and see results.
+All other views are unchanged from the non-enhanced version. You can still access the original <strong>Query</strong> view by navigating to <code>#query</code> in the URL.
+You can tell that you're looking at the updated <strong>Query</strong> view by the presence of the tabs (3).</li>
+<li>The <strong>druid</strong> panel shows the available schemas, datasources, and columns.</li>
+<li>Query tabs allow you to manage and run several queries at once.
+Click the plus icon to open a new tab.
+To manipulate existing tabs, click the tab name.</li>
+<li>The tab bar contains some helpful tools including the <strong>Connect external data</strong> button that samples external data and creates an initial query with the appropriate <code>EXTERN</code> definition that you can then edit as needed.</li>
+<li>The <strong>Recent query tasks</strong> panel lets you see currently running and previous queries from all users in the cluster.
+It is equivalent to the <strong>Task</strong> view in the <strong>Ingestion</strong> view with the filter of <code>type='query_controller'</code>.</li>
+<li>You can click on each query entry to attach to that query in a new tab.</li>
+<li>You can download an archive of all the pertinent details about the query that you can share.</li>
+<li>The <strong>Run</strong> button runs the query.</li>
+<li>The <strong>Preview</strong> button appears when you enter an INSERT/REPLACE query. It runs the query inline without the INSERT/REPLACE clause and with an added LIMIT to give you a preview of the data that would be ingested if you click <strong>Run</strong>.
+The added LIMIT makes the query run faster but provides incomplete results.</li>
+<li>The engine selector lets you choose which engine (API endpoint) to send a query to. By default, it automatically picks which endpoint to use based on an analysis of the query, but you can select a specific engine explicitly. You can also configure the engine specific context parameters from this menu.</li>
+<li>The <strong>Max tasks</strong> picker appears when you have the <strong>sql-msq-task</strong> engine selected. It lets you configure the degree of parallelism.</li>
+<li>The More menu (<strong>...</strong>) contains the following helpful tools:</li>
+</ol>
+<ul>
+<li><strong>Explain SQL query</strong> shows you the logical plan returned by <code>EXPLAIN PLAN FOR</code> for a SQL query.</li>
+<li><strong>Query history</strong> shows you previously executed queries.</li>
+<li><strong>Convert ingestion spec to SQL</strong> lets you convert a native batch ingestion spec to an equivalent SQL query.</li>
+<li><strong>Attach tab from task ID</strong> lets you create a new tab from the task ID of a query executed on this cluster.</li>
+<li><strong>Open query detail archive</strong> lets you open a detail archive generated on any cluster by (7).</li>
+</ul>
+<ol start="13">
+<li>The query timer indicates how long the query has been running for.</li>
+<li>The <strong>(cancel)</strong> link cancels the currently running query.</li>
+<li>The main progress bar shows the overall progress of the query.
+The progress is computed from the various counters in the live reports (16).</li>
+<li>The <strong>Current stage</strong> progress bar shows the progress for the currently running query stage.
+If several stages are executing concurrently, it conservatively shows the information for the earliest executing stage.</li>
+<li>The live query reports show detailed information of all the stages (past, present, and future). The live reports are shown while the query is running. You can hide the report if you want.
+After queries finish, you can access them by clicking on the query time indicator or from the <strong>Recent query tasks</strong> panel (6).</li>
+<li>You can expand each stage of the live query report by clicking on the triangle to show per worker and per partition statistics.</li>
+</ol>
+<h2><a class="anchor" aria-hidden="true" id="data-loader"></a><a href="#data-loader" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>You can use the data loader to build an ingestion spec with a step-by-step wizard.</p>
+<p><img src="../assets/web-console-02-data-loader-1.png" alt="data-loader-1"></p>
+<p>After selecting the location of your data, follow the series of steps displaying incremental previews of the data as it is ingested.
+After filling in the required details on every step you can navigate to the next step by clicking <strong>Next</strong>.
+You can also freely navigate between the steps from the top navigation.</p>
+<p>Navigating with the top navigation leaves the underlying spec unmodified while clicking <strong>Next</strong> attempts to fill in the subsequent steps with appropriate defaults.</p>
+<p><img src="../assets/web-console-03-data-loader-2.png" alt="data-loader-2"></p>
+<h2><a class="anchor" aria-hidden="true" id="datasources"></a><a href="#datasources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>The <strong>Datasources</strong> view shows all the datasources currently loaded on the cluster, as well as their sizes and availability.
+From the <strong>Datasources</strong> view, you can edit the retention rules, configure automatic compaction, and drop data in a datasource.</p>
+<p>A datasource is partitioned into one or more segments organized by time chunks.
+To display a timeline of segments, toggle the option for <strong>Show segment timeline</strong>.</p>
+<p>Like any view that is powered by a Druid SQL query, you can click <strong>View SQL query for table</strong> from the ellipsis menu to run the underlying SQL query directly.</p>
+<p><img src="../assets/web-console-04-datasources.png" alt="datasources"></p>
+<p>You can view and edit retention rules to determine the general availability of a datasource.</p>
+<p><img src="../assets/web-console-05-retention.png" alt="retention"></p>
+<h2><a class="anchor" aria-hidden="true" id="segments"></a><a href="#segments" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
+<p>The <strong>Segments</strong> view shows all the <a href="/docs/24.0.0/design/segments.html">segments</a> in the cluster.
+Each segment has a detail view that provides more information.
+The Segment ID is also conveniently broken down into Datasource, Start, End, Version, and Partition columns for ease of filtering and sorting.</p>
+<p><img src="../assets/web-console-06-segments.png" alt="segments"></p>
+<h2><a class="anchor" aria-hidden="true" id="supervisors-and-tasks"></a><a href="#supervisors-and-tasks" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>From this view, you can check the status of existing supervisors as well as suspend, resume, and reset them.
+The supervisor oversees the state of the indexing tasks to coordinate handoffs, manage failures, and ensure that the scalability and replication requirements are maintained.</p>
+<p>The tasks table allows you to see the currently running and recently completed tasks.
+To navigate your tasks more easily, you can group them by their <strong>Type</strong>, <strong>Datasource</strong>, or <strong>Status</strong>.
+Submit a task manually by clicking the ellipsis icon and selecting <strong>Submit JSON task</strong>.</p>
+<p><img src="../assets/web-console-07-supervisors.png" alt="supervisors"></p>
+<p>Click on the magnifying glass for any supervisor to see detailed reports of its progress.</p>
+<p><img src="../assets/web-console-08-supervisor-status.png" alt="supervisor-status"></p>
+<p>Click on the magnifying glass for any task to see more detail about it.</p>
+<p><img src="../assets/web-console-09-task-status.png" alt="tasks-status"></p>
+<h2><a class="anchor" aria-hidden="true" id="services"></a><a href="#services" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
+<p>The <strong>Services</strong> view lets you see the current status of the nodes making up your cluster.
+You can group the nodes by type or by tier to get meaningful summary statistics.</p>
+<p><img src="../assets/web-console-10-servers.png" alt="servers"></p>
+<h2><a class="anchor" aria-hidden="true" id="lookups"></a><a href="#lookups" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<p>Access the <strong>Lookups</strong> view from the <strong>Lookups</strong> card in the home view or by clicking on the gear icon in the upper right corner.
+Here you can create and edit query time <a href="/docs/24.0.0/querying/lookups.html">lookups</a>.</p>
+<p><img src="../assets/web-console-13-lookups.png" alt="lookups"></p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/configuration/logging.html"><span class="arrow-prev">← </span><span>Logging</span></a><a class="docs-next button" href="/docs/24.0.0/operations/java.html"><span>Java runtime</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#home">Home</a></li><li><a href="#query">Query</a></li><li><a href="#data-loader">Data loade [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/querying/aggregations.html b/docs/24.0.0/querying/aggregations.html
index 66e05f43..f6b54fd9 100644
--- a/docs/24.0.0/querying/aggregations.html
+++ b/docs/24.0.0/querying/aggregations.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/caching.html b/docs/24.0.0/querying/caching.html
index d952f749..87c9ab96 100644
--- a/docs/24.0.0/querying/caching.html
+++ b/docs/24.0.0/querying/caching.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/datasource.html b/docs/24.0.0/querying/datasource.html
index 9c474193..a1e0067c 100644
--- a/docs/24.0.0/querying/datasource.html
+++ b/docs/24.0.0/querying/datasource.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/datasourcemetadataquery.html b/docs/24.0.0/querying/datasourcemetadataquery.html
index 29674104..36c8929f 100644
--- a/docs/24.0.0/querying/datasourcemetadataquery.html
+++ b/docs/24.0.0/querying/datasourcemetadataquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/dimensionspecs.html b/docs/24.0.0/querying/dimensionspecs.html
index e1fb2bf7..8d4860ef 100644
--- a/docs/24.0.0/querying/dimensionspecs.html
+++ b/docs/24.0.0/querying/dimensionspecs.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -126,9 +126,6 @@ language. For information about functions available in SQL, refer to the
</code></pre>
<p>For more details and examples, see <a href="/docs/24.0.0/querying/multi-value-dimensions.html">multi-value dimensions</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="lookup-dimensionspecs"></a><a href="#lookup-dimensionspecs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
-<blockquote>
-<p>Lookups are an <a href="/docs/24.0.0/development/experimental.html">experimental</a> feature.</p>
-</blockquote>
<p>You can use lookup dimension specs to define a lookup implementation as a dimension spec directly.
Generally, there are two kinds of lookup implementations.
The first kind is passed at the query time like <code>map</code> implementation.</p>
diff --git a/docs/24.0.0/querying/filters.html b/docs/24.0.0/querying/filters.html
index a29bb3dc..d3bb9e15 100644
--- a/docs/24.0.0/querying/filters.html
+++ b/docs/24.0.0/querying/filters.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/granularities.html b/docs/24.0.0/querying/granularities.html
index fece67da..b6cb76eb 100644
--- a/docs/24.0.0/querying/granularities.html
+++ b/docs/24.0.0/querying/granularities.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/groupbyquery.html b/docs/24.0.0/querying/groupbyquery.html
index 9eee353e..8978dc0c 100644
--- a/docs/24.0.0/querying/groupbyquery.html
+++ b/docs/24.0.0/querying/groupbyquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/having.html b/docs/24.0.0/querying/having.html
index 8f280d27..279c13a1 100644
--- a/docs/24.0.0/querying/having.html
+++ b/docs/24.0.0/querying/having.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/hll-old.html b/docs/24.0.0/querying/hll-old.html
index d996b375..bf21653c 100644
--- a/docs/24.0.0/querying/hll-old.html
+++ b/docs/24.0.0/querying/hll-old.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/joins.html b/docs/24.0.0/querying/joins.html
index 7542303d..36987740 100644
--- a/docs/24.0.0/querying/joins.html
+++ b/docs/24.0.0/querying/joins.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/limitspec.html b/docs/24.0.0/querying/limitspec.html
index 0b3840bf..ea17f2ea 100644
--- a/docs/24.0.0/querying/limitspec.html
+++ b/docs/24.0.0/querying/limitspec.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/lookups.html b/docs/24.0.0/querying/lookups.html
index 1fc0eeba..62d1ec73 100644
--- a/docs/24.0.0/querying/lookups.html
+++ b/docs/24.0.0/querying/lookups.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -447,11 +447,7 @@ The return value will be the json representation of the factory.</p>
<span class="hljs-string">"Dash"</span>
]
</code></pre>
-<h2><a class="anchor" aria-hidden="true" id="druid-version-0100-to-0101-upgradedowngrade"></a><a href="#druid-version-0100-to-0101-upgradedowngrade" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h [...]
-<p>Overall druid cluster lookups configuration is persisted in metadata store and also individual lookup processes optionally persist a snapshot of loaded lookups on disk.
-If upgrading from druid version 0.10.0 to 0.10.1, then migration for all persisted metadata is handled automatically.
-If downgrading from 0.10.1 to 0.9.0 then lookups updates done via Coordinator while 0.10.1 was running, would be lost.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/joins.html"><span class="arrow-prev">← </span><span>Joins</span></a><a class="docs-next button" href="/docs/24.0.0/querying/multi-value-dimensions.html"><span>Multi-value dimensions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-syntax">Query Syntax</a></li><li><a href="#query-execution">Query Ex [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/joins.html"><span class="arrow-prev">← </span><span>Joins</span></a><a class="docs-next button" href="/docs/24.0.0/querying/multi-value-dimensions.html"><span>Multi-value dimensions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-syntax">Query Syntax</a></li><li><a href="#query-execution">Query Ex [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/multi-value-dimensions.html b/docs/24.0.0/querying/multi-value-dimensions.html
index d625af5f..cf898f6d 100644
--- a/docs/24.0.0/querying/multi-value-dimensions.html
+++ b/docs/24.0.0/querying/multi-value-dimensions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/multitenancy.html b/docs/24.0.0/querying/multitenancy.html
index bd5fcc91..22afbc64 100644
--- a/docs/24.0.0/querying/multitenancy.html
+++ b/docs/24.0.0/querying/multitenancy.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/nested-columns.html b/docs/24.0.0/querying/nested-columns.html
index 4dad268a..e8f6b717 100644
--- a/docs/24.0.0/querying/nested-columns.html
+++ b/docs/24.0.0/querying/nested-columns.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/post-aggregations.html b/docs/24.0.0/querying/post-aggregations.html
index 5eea928e..177266e3 100644
--- a/docs/24.0.0/querying/post-aggregations.html
+++ b/docs/24.0.0/querying/post-aggregations.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/query-context.html b/docs/24.0.0/querying/query-context.html
index 589459df..1621840f 100644
--- a/docs/24.0.0/querying/query-context.html
+++ b/docs/24.0.0/querying/query-context.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/query-execution.html b/docs/24.0.0/querying/query-execution.html
index ea0f3f99..b4089771 100644
--- a/docs/24.0.0/querying/query-execution.html
+++ b/docs/24.0.0/querying/query-execution.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/querying.html b/docs/24.0.0/querying/querying.html
index 1407c863..1d45715f 100644
--- a/docs/24.0.0/querying/querying.html
+++ b/docs/24.0.0/querying/querying.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -90,7 +90,7 @@ posted like this:</p>
<blockquote>
<p>Replace <code><queryable_host>:<port></code> with the appropriate address and port for your system. For example, if running the quickstart configuration, replace <code><queryable_host>:<port></code> with localhost:8888.</p>
</blockquote>
-<p>You can also enter them directly in the Druid console's Query view. Simply pasting a native query into the console switches the editor into JSON mode.</p>
+<p>You can also enter them directly in the web console's Query view. Simply pasting a native query into the console switches the editor into JSON mode.</p>
<p><img src="../assets/native-queries-01.png" alt="Native query" title="Native query"></p>
<p>Druid's native query language is JSON over HTTP, although many members of the community have contributed different
<a href="https://druid.apache.org/libraries.html">client libraries</a> in other languages to query Druid.</p>
diff --git a/docs/24.0.0/querying/scan-query.html b/docs/24.0.0/querying/scan-query.html
index 2b61145e..617f6765 100644
--- a/docs/24.0.0/querying/scan-query.html
+++ b/docs/24.0.0/querying/scan-query.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/searchquery.html b/docs/24.0.0/querying/searchquery.html
index 89202b74..cedbe30f 100644
--- a/docs/24.0.0/querying/searchquery.html
+++ b/docs/24.0.0/querying/searchquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/segmentmetadataquery.html b/docs/24.0.0/querying/segmentmetadataquery.html
index f689e11e..d3f9bfc7 100644
--- a/docs/24.0.0/querying/segmentmetadataquery.html
+++ b/docs/24.0.0/querying/segmentmetadataquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/select-query.html b/docs/24.0.0/querying/select-query.html
index 85c3fe09..3c879e74 100644
--- a/docs/24.0.0/querying/select-query.html
+++ b/docs/24.0.0/querying/select-query.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -78,7 +78,7 @@
~ under the License.
-->
<p>Older versions of Apache Druid included a Select query type. Since Druid 0.17.0, it has been removed and replaced by the <a href="/docs/24.0.0/querying/scan-query.html">Scan query</a>, which offers improved memory usage and performance. This solves issues that users had with Select queries causing Druid to run out of memory or slow down.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/hll-old.html"><span class="arrow-prev">← </span><span class="function-name-prevnext">Cardinality/HyperUnique aggregators</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/standalone-realtime.html"><span>Realtime Process</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="fo [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/hll-old.html"><span class="arrow-prev">← </span><span class="function-name-prevnext">Cardinality/HyperUnique aggregators</span></a><a class="docs-next button" href="/docs/24.0.0/ingestion/native-batch-firehose.html"><span>Firehose (deprecated)</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/sorting-orders.html b/docs/24.0.0/querying/sorting-orders.html
index dbaca359..0e814be5 100644
--- a/docs/24.0.0/querying/sorting-orders.html
+++ b/docs/24.0.0/querying/sorting-orders.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-aggregations.html b/docs/24.0.0/querying/sql-aggregations.html
index b80fe49c..8db76299 100644
--- a/docs/24.0.0/querying/sql-aggregations.html
+++ b/docs/24.0.0/querying/sql-aggregations.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -117,12 +117,12 @@ consider using the <code>ROUND</code> function to smooth out the inconsistencies
</thead>
<tbody>
<tr><td><code>COUNT(*)</code></td><td>Counts the number of rows.</td><td><code>0</code></td></tr>
-<tr><td><code>COUNT(DISTINCT expr)</code></td><td>Counts distinct values of <code>expr</code>.<br><br>When <code>useApproximateCountDistinct</code> is set to "true" (the default), this is an alias for <code>APPROX_COUNT_DISTINCT</code>. The specific algorithm depends on the value of <a href="/docs/24.0.0/configuration/index.html#sql"><code>druid.sql.approxCountDistinct.function</code></a>. In this mode, you can use strings, numbers, or prebuilt sketches. If counting prebuilt sk [...]
+<tr><td><code>COUNT(DISTINCT expr)</code></td><td>Counts distinct values of <code>expr</code>.<br /><br />When <code>useApproximateCountDistinct</code> is set to "true" (the default), this is an alias for <code>APPROX_COUNT_DISTINCT</code>. The specific algorithm depends on the value of <a href="/docs/24.0.0/configuration/index.html#sql"><code>druid.sql.approxCountDistinct.function</code></a>. In this mode, you can use strings, numbers, or prebuilt sketches. If counting prebuil [...]
<tr><td><code>SUM(expr)</code></td><td>Sums numbers.</td><td><code>null</code> if <code>druid.generic.useDefaultValueForNull=false</code>, otherwise <code>0</code></td></tr>
<tr><td><code>MIN(expr)</code></td><td>Takes the minimum of numbers.</td><td><code>null</code> if <code>druid.generic.useDefaultValueForNull=false</code>, otherwise <code>9223372036854775807</code> (maximum LONG value)</td></tr>
<tr><td><code>MAX(expr)</code></td><td>Takes the maximum of numbers.</td><td><code>null</code> if <code>druid.generic.useDefaultValueForNull=false</code>, otherwise <code>-9223372036854775808</code> (minimum LONG value)</td></tr>
<tr><td><code>AVG(expr)</code></td><td>Averages numbers.</td><td><code>null</code> if <code>druid.generic.useDefaultValueForNull=false</code>, otherwise <code>0</code></td></tr>
-<tr><td><code>APPROX_COUNT_DISTINCT(expr)</code></td><td>Counts distinct values of <code>expr</code> using an approximate algorithm. The <code>expr</code> can be a regular column or a prebuilt sketch column.<br><br>The specific algorithm depends on the value of <a href="/docs/24.0.0/configuration/index.html#sql"><code>druid.sql.approxCountDistinct.function</code></a>. By default, this is <code>APPROX_COUNT_DISTINCT_BUILTIN</code>. If the <a href="/docs/24.0.0/development/extensions-core/ [...]
+<tr><td><code>APPROX_COUNT_DISTINCT(expr)</code></td><td>Counts distinct values of <code>expr</code> using an approximate algorithm. The <code>expr</code> can be a regular column or a prebuilt sketch column.<br /><br />The specific algorithm depends on the value of <a href="/docs/24.0.0/configuration/index.html#sql"><code>druid.sql.approxCountDistinct.function</code></a>. By default, this is <code>APPROX_COUNT_DISTINCT_BUILTIN</code>. If the <a href="/docs/24.0.0/development/extensions-c [...]
<tr><td><code>APPROX_COUNT_DISTINCT_BUILTIN(expr)</code></td><td><em>Usage note:</em> consider using <code>APPROX_COUNT_DISTINCT_DS_HLL</code> instead, which offers better accuracy in many cases.<br/><br/>Counts distinct values of <code>expr</code> using Druid's built-in "cardinality" or "hyperUnique" aggregators, which implement a variant of <a href="http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf">HyperLogLog</a>. The <code>expr</code> can be a string, a n [...]
<tr><td><code>APPROX_QUANTILE(expr, probability, [resolution])</code></td><td><em>Deprecated.</em> Use <code>APPROX_QUANTILE_DS</code> instead, which provides a superior distribution-independent algorithm with formal error guarantees.<br/><br/>Computes approximate quantiles on numeric or <a href="/docs/24.0.0/development/extensions-core/approximate-histograms.html#approximate-histogram-aggregator">approxHistogram</a> expressions. <code>probability</code> should be between 0 and 1, exclus [...]
<tr><td><code>APPROX_QUANTILE_FIXED_BUCKETS(expr, probability, numBuckets, lowerLimit, upperLimit, [outlierHandlingMode])</code></td><td>Computes approximate quantiles on numeric or <a href="/docs/24.0.0/development/extensions-core/approximate-histograms.html#fixed-buckets-histogram">fixed buckets histogram</a> expressions. <code>probability</code> should be between 0 and 1, exclusive. The <code>numBuckets</code>, <code>lowerLimit</code>, <code>upperLimit</code>, and <code>outlierHandlin [...]
diff --git a/docs/24.0.0/querying/sql-api.html b/docs/24.0.0/querying/sql-api.html
index 72bd30ee..e2d3263b 100644
--- a/docs/24.0.0/querying/sql-api.html
+++ b/docs/24.0.0/querying/sql-api.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -210,7 +210,7 @@ delivered due to an error.</p>
<li>The query completes before your cancellation request is processed.</li>
</ul>
<p>Druid returns an HTTP 403 response for authorization failure.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-json-functions.html"><span class="arrow-prev">← </span><span>JSON functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-jdbc.html"><span>JDBC driver API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-a-query">Submit a query</a><ul class="toc-headings"><li><a href="# [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-functions.html"><span class="arrow-prev">← </span><span>All functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-jdbc.html"><span>JDBC driver API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#submit-a-query">Submit a query</a><ul class="toc-headings"><li><a href="#reques [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/sql-data-types.html b/docs/24.0.0/querying/sql-data-types.html
index ac118f12..eee191cf 100644
--- a/docs/24.0.0/querying/sql-data-types.html
+++ b/docs/24.0.0/querying/sql-data-types.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-functions.html b/docs/24.0.0/querying/sql-functions.html
index 2dbcf8fe..52bf42a7 100644
--- a/docs/24.0.0/querying/sql-functions.html
+++ b/docs/24.0.0/querying/sql-functions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -761,7 +761,7 @@ Click the linked function type for documentation on a particular function.</p>
<p><code>VARIANCE(expr)</code></p>
<p><strong>Function type:</strong> <a href="/docs/24.0.0/querying/sql-aggregations.html">Aggregation</a></p>
<p>Alias for <a href="#var_samp"><code>VAR_SAMP</code></a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-multivalue-string-functions.html"><span class="arrow-prev">← </span><span>Multi-value string functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-json-functions.html"><span>JSON functions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#abs">ABS</a></li><li><a href="#acos"> [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-json-functions.html"><span class="arrow-prev">← </span><span>JSON functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-api.html"><span>Druid SQL API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#abs">ABS</a></li><li><a href="#acos">ACOS</a></li><li><a href="#any_value">A [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/sql-jdbc.html b/docs/24.0.0/querying/sql-jdbc.html
index b9b5ab16..54219b4a 100644
--- a/docs/24.0.0/querying/sql-jdbc.html
+++ b/docs/24.0.0/querying/sql-jdbc.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-json-functions.html b/docs/24.0.0/querying/sql-json-functions.html
index e67470c8..d6c0aadd 100644
--- a/docs/24.0.0/querying/sql-json-functions.html
+++ b/docs/24.0.0/querying/sql-json-functions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -117,16 +117,16 @@
<pre><code class="hljs css language-json">{<span class="hljs-attr">"x"</span>:<span class="hljs-number">1</span>, <span class="hljs-attr">"y"</span>:[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>]}
</code></pre>
<ul>
-<li>To return the entire JSON object:<br>
+<li>To return the entire JSON object:<br />
<code>$</code> -> <code>{"x":1, "y":[1, 2, 3]}</code></li>
-<li>To return the value of the key "x":<br>
+<li>To return the value of the key "x":<br />
<code>$.x</code> -> <code>1</code></li>
-<li>For a key that contains an array, to return the entire array:<br>
+<li>For a key that contains an array, to return the entire array:<br />
<code>$['y']</code> -> <code>[1, 2, 3]</code></li>
-<li>For a key that contains an array, to return an item in the array:<br>
+<li>For a key that contains an array, to return an item in the array:<br />
<code>$.y[1]</code> -> <code>2</code></li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-functions.html"><span class="arrow-prev">← </span><span>All functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-api.html"><span>Druid SQL API</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a hre [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-multivalue-string-functions.html"><span class="arrow-prev">← </span><span>Multi-value string functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-functions.html"><span>All functions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="contai [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/sql-metadata-tables.html b/docs/24.0.0/querying/sql-metadata-tables.html
index 8133754e..e5457725 100644
--- a/docs/24.0.0/querying/sql-metadata-tables.html
+++ b/docs/24.0.0/querying/sql-metadata-tables.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-multivalue-string-functions.html b/docs/24.0.0/querying/sql-multivalue-string-functions.html
index fd2b0d23..4d7863f6 100644
--- a/docs/24.0.0/querying/sql-multivalue-string-functions.html
+++ b/docs/24.0.0/querying/sql-multivalue-string-functions.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -115,7 +115,7 @@ See <a href="/docs/24.0.0/querying/multi-value-dimensions.html">Multi-value dime
<tr><td><code>STRING_TO_MV(str1, str2)</code></td><td>Splits <code>str1</code> into an array on the delimiter specified by <code>str2</code>.</td></tr>
</tbody>
</table>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-aggregations.html"><span class="arrow-prev">← </span><span>Aggregation functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-functions.html"><span>All functions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text- [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/querying/sql-aggregations.html"><span class="arrow-prev">← </span><span>Aggregation functions</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-json-functions.html"><span>JSON functions</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class= [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/sql-operators.html b/docs/24.0.0/querying/sql-operators.html
index 74e166e0..807c4501 100644
--- a/docs/24.0.0/querying/sql-operators.html
+++ b/docs/24.0.0/querying/sql-operators.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-query-context.html b/docs/24.0.0/querying/sql-query-context.html
index 2e10c183..65a1d4da 100644
--- a/docs/24.0.0/querying/sql-query-context.html
+++ b/docs/24.0.0/querying/sql-query-context.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-scalar.html b/docs/24.0.0/querying/sql-scalar.html
index 33f5af77..fd97cf75 100644
--- a/docs/24.0.0/querying/sql-scalar.html
+++ b/docs/24.0.0/querying/sql-scalar.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql-translation.html b/docs/24.0.0/querying/sql-translation.html
index 2fb00a81..3df49347 100644
--- a/docs/24.0.0/querying/sql-translation.html
+++ b/docs/24.0.0/querying/sql-translation.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/sql.html b/docs/24.0.0/querying/sql.html
index b6b86dc4..acda9381 100644
--- a/docs/24.0.0/querying/sql.html
+++ b/docs/24.0.0/querying/sql.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -245,7 +245,7 @@ which they are passed. Parameters are supported in both the <a href="/docs/24.0.
<p>To solve this issue, explicitly provide the type of the dynamic parameter using the <code>CAST</code> keyword. Consider the fix for the preceding example:</p>
<pre><code class="hljs"><span class="hljs-keyword">SELECT</span> * <span class="hljs-keyword">FROM</span> druid.foo <span class="hljs-keyword">WHERE</span> dim1 <span class="hljs-keyword">like</span> CONCAT(<span class="hljs-string">'%'</span>, CAST (? <span class="hljs-keyword">AS</span> <span class="hljs-type">VARCHAR</span>), <span class="hljs-string">'%'</span>)
</code></pre>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/multi-stage-query/known-issues.html"><span class="arrow-prev">← </span><span>Known issues</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-data-types.html"><span>SQL data types</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#syntax">Syntax</a></li><li><a href="#from">FROM</a></li><li><a href [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/data-management/automatic-compaction.html"><span class="arrow-prev">← </span><span>Automatic compaction</span></a><a class="docs-next button" href="/docs/24.0.0/querying/sql-data-types.html"><span>SQL data types</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#syntax">Syntax</a></li><li><a href="#from">FROM</a></ [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/querying/timeboundaryquery.html b/docs/24.0.0/querying/timeboundaryquery.html
index 3d806ece..90cb6567 100644
--- a/docs/24.0.0/querying/timeboundaryquery.html
+++ b/docs/24.0.0/querying/timeboundaryquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/timeseriesquery.html b/docs/24.0.0/querying/timeseriesquery.html
index dcd1b277..21bf7b78 100644
--- a/docs/24.0.0/querying/timeseriesquery.html
+++ b/docs/24.0.0/querying/timeseriesquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/topnmetricspec.html b/docs/24.0.0/querying/topnmetricspec.html
index 48b9fbce..f9f7581d 100644
--- a/docs/24.0.0/querying/topnmetricspec.html
+++ b/docs/24.0.0/querying/topnmetricspec.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/topnquery.html b/docs/24.0.0/querying/topnquery.html
index 06c13bee..e1e8f358 100644
--- a/docs/24.0.0/querying/topnquery.html
+++ b/docs/24.0.0/querying/topnquery.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/troubleshooting.html b/docs/24.0.0/querying/troubleshooting.html
index db259333..5d0aa551 100644
--- a/docs/24.0.0/querying/troubleshooting.html
+++ b/docs/24.0.0/querying/troubleshooting.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/using-caching.html b/docs/24.0.0/querying/using-caching.html
index f7cdb9ec..5c05bf06 100644
--- a/docs/24.0.0/querying/using-caching.html
+++ b/docs/24.0.0/querying/using-caching.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/querying/virtual-columns.html b/docs/24.0.0/querying/virtual-columns.html
index c267f146..698ff029 100644
--- a/docs/24.0.0/querying/virtual-columns.html
+++ b/docs/24.0.0/querying/virtual-columns.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/tutorials/cluster.html b/docs/24.0.0/tutorials/cluster.html
index 3cb419f8..985b3ae1 100644
--- a/docs/24.0.0/tutorials/cluster.html
+++ b/docs/24.0.0/tutorials/cluster.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -389,7 +389,7 @@ This also allows you take advantage of Druid's built-in MiddleManager autoscalin
<h2><a class="anchor" aria-hidden="true" id="loading-data"></a><a href="#loading-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
<p>Congratulations, you now have a Druid cluster! The next step is to learn about recommended ways to load data into
Druid based on your use case. Read more about <a href="/docs/24.0.0/ingestion/index.html">loading data</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/single-server.html"><span class="arrow-prev">← </span><span>Single server deployment</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-batch.html"><span>Loading files natively</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#select-hardware">Select hardware</a><ul class="toc-h [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/operations/single-server.html"><span class="arrow-prev">← </span><span>Single server deployment</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-batch.html"><span>Load files natively</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#select-hardware">Select hardware</a><ul class="toc-head [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/docker.html b/docs/24.0.0/tutorials/docker.html
index 8e3e814f..0149b18c 100644
--- a/docs/24.0.0/tutorials/docker.html
+++ b/docs/24.0.0/tutorials/docker.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Docker · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/docker.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><meta property=" [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial: Run with Docker · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/docker.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/docker.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Docker</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/docker.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Tutorial: Run with Docker</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -134,8 +134,8 @@
<h2><a class="anchor" aria-hidden="true" id="launching-the-cluster"></a><a href="#launching-the-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
<p><code>cd</code> into the directory that contains the configuration files. This is the directory you created above, or the <code>distribution/docker/</code> in your Druid installation directory if you installed Druid locally.</p>
<p>Run <code>docker-compose up</code> to launch the cluster with a shell attached, or <code>docker-compose up -d</code> to run the cluster in the background.</p>
-<p>Once the cluster has started, you can navigate to the <a href="/docs/24.0.0/operations/druid-console.html">Druid console</a> at <a href="http://localhost:8888">http://localhost:8888</a>. The <a href="/docs/24.0.0/design/router.html">Druid router process</a> serves the UI.</p>
-<p><img src="../assets/tutorial-quickstart-01.png" alt="Druid console" title="Druid console"></p>
+<p>Once the cluster has started, you can navigate to the <a href="/docs/24.0.0/operations/web-console.html">web console</a> at <a href="http://localhost:8888">http://localhost:8888</a>. The <a href="/docs/24.0.0/design/router.html">Druid router process</a> serves the UI.</p>
+<p><img src="../assets/tutorial-quickstart-01.png" alt="web console" title="web console"></p>
<p>It takes a few seconds for all the Druid processes to fully start up. If you open the console immediately after starting the services, you may see some errors that you can safely ignore.</p>
<h2><a class="anchor" aria-hidden="true" id="using-the-cluster"></a><a href="#using-the-cluster" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
<p>From here you can follow along with the <a href="/docs/24.0.0/tutorials/index.html#load-data">Quickstart</a>. For production use, refine your <code>docker-compose.yml</code> file to add any additional external service dependencies as necessary.</p>
@@ -144,7 +144,7 @@
</code></pre>
<p>Where <code><id></code> is the container id found with <code>docker ps</code>. Druid is installed in <code>/opt/druid</code>. The <a href="https://github.com/apache/druid/blob/24.0.0/distribution/docker/druid.sh">script</a> which consumes the environment variables mentioned above, and which launches Druid, is located at <code>/druid.sh</code>.</p>
<p>Run <code>docker-compose down</code> to shut down the cluster. Your data is persisted as a set of <a href="https://docs.docker.com/storage/volumes/">Docker volumes</a> and will be available when you restart your Druid cluster.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/index.html"><span class="arrow-prev">← </span><span>Quickstart</span></a><a class="docs-next button" href="/docs/24.0.0/operations/single-server.html"><span>Single server deployment</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#prerequisites">Prerequisites</a><ul class="toc-headings"><li><a href="#do [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-transform-spec.html"><span class="arrow-prev">← </span><span>Transforming input data</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html"><span>Kerberized HDFS deep storage</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#prerequisites">Prerequisites [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/index.html b/docs/24.0.0/tutorials/index.html
index 64b0941a..33ad5852 100644
--- a/docs/24.0.0/tutorials/index.html
+++ b/docs/24.0.0/tutorials/index.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Quickstart · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><meta propert [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Quickstart (local) · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><meta [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Quickstart</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Quickstart (local)</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -77,37 +77,34 @@
~ specific language governing permissions and limitations
~ under the License.
-->
-<p>This quickstart gets you started with Apache Druid using the <code>micro-quickstart</code> startup configuration and introduces you to some Druid features, including the MSQ task engine that's part of the <a href="/docs/24.0.0/multi-stage-query/index.html">multi-stage query architecture</a>.</p>
-<p>With the MSQ task engine, you can write query tasks that can reference <a href="/docs/24.0.0/multi-stage-query/index.html#read-external-data">external data</a> as well as perform ingestion with SQL <a href="/docs/24.0.0/multi-stage-query/index.html#insert-data">INSERT</a> and <a href="/docs/24.0.0/multi-stage-query/index.html#replace-data">REPLACE</a>, eliminating the need to generate JSON-based ingestion specs.</p>
+<p>This quickstart gets you started with Apache Druid using the <a href="/docs/24.0.0/operations/single-server.html#micro-quickstart-4-cpu-16gib-ram"><code>micro-quickstart</code></a> configuration, and introduces you to Druid ingestion and query features.</p>
<p>In this quickstart, you'll do the following:</p>
<ul>
<li>install Druid</li>
<li>start up Druid services</li>
-<li>use the MSQ task engine to ingest data</li>
+<li>use SQL to ingest and query data</li>
</ul>
-<p>Druid supports different ingestion engines. While we recommend SQL based ingestion, you can find tutorials for other modes of ingestion, such as <a href="/docs/24.0.0/tutorials/tutorial-batch-native.html">Load data with native batch ingestion</a>.</p>
+<p>Druid supports a variety of ingestion options. Once you're done with this tutorial, refer to the
+<a href="/docs/24.0.0/ingestion/index.html">Ingestion</a> page to determine which ingestion method is right for you.</p>
<h2><a class="anchor" aria-hidden="true" id="requirements"></a><a href="#requirements" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
-<p>You can follow these steps on a relatively small machine, such as a laptop with around 4 CPU and 16 GiB of RAM.</p>
-<p>Druid comes equipped with several startup configuration profiles for a range of machine sizes.</p>
-<p>The <code>micro-quickstart</code> configuration profile is suitable for evaluating Druid. If you want to
-try out Druid's performance or scaling capabilities, you'll need a larger machine and configuration profile.</p>
-<p>The configuration profiles included with Druid range from the even smaller <em>Nano-Quickstart</em> configuration (1 CPU, 4GiB RAM)
-to the <em>X-Large</em> configuration (64 CPU, 512GiB RAM). For more information, see <a href="/docs/24.0.0/operations/single-server.html">Single server deployment</a>.
-For information on deploying Druid services across clustered machines, see <a href="/docs/24.0.0/tutorials/cluster.html">Clustered deployment</a>.</p>
+<p>You can follow these steps on a relatively modest machine, such as a workstation or virtual server with 16 GiB of RAM.</p>
+<p>Druid comes equipped with several <a href="/docs/24.0.0/operations/single-server.html">startup configuration profiles</a> for a
+range of machine sizes. These range from <code>nano</code> (1 CPU, 4GiB RAM) to <code>x-large</code> (64 CPU, 512GiB RAM). For more
+information, see <a href="/docs/24.0.0/operations/single-server.html">Single server deployment</a>. For information on deploying Druid services
+across clustered machines, see <a href="/docs/24.0.0/tutorials/cluster.html">Clustered deployment</a>.</p>
<p>The software requirements for the installation machine are:</p>
<ul>
-<li>Linux, Mac OS X, or other Unix-like OS (Windows is not supported)</li>
-<li>Java 8, Update 92 or later (8u92+) or Java 11</li>
+<li>Linux, Mac OS X, or other Unix-like OS. (Windows is not supported.)</li>
+<li>Java 8u92+ or Java 11.</li>
</ul>
<blockquote>
<p>Druid relies on the environment variables <code>JAVA_HOME</code> or <code>DRUID_JAVA_HOME</code> to find Java on the machine. You can set
<code>DRUID_JAVA_HOME</code> if there is more than one instance of Java. To verify Java requirements for your environment, run the
<code>bin/verify-java</code> script.</p>
</blockquote>
-<p>Before installing a production Druid instance, be sure to consider the user account on the operating system under
-which Druid will run. This is important because any Druid console user will have, effectively, the same permissions as
-that user. For example, the file browser UI will show console users the files that the underlying user can
-access. In general, avoid running Druid as root user. Consider creating a dedicated user account for running Druid.</p>
+<p>Before installing a production Druid instance, be sure to review the <a href="/docs/24.0.0/operations/security-overview.html">security
+overview</a>. In general, avoid running Druid as root user. Consider creating a
+dedicated user account for running Druid.</p>
<h2><a class="anchor" aria-hidden="true" id="install-druid"></a><a href="#install-druid" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
<p>Download the <a href="https://www.apache.org/dyn/closer.cgi?path=/druid/24.0.0/apache-druid-24.0.0-bin.tar.gz">24.0.0 release</a> from Apache Druid.
For this quickstart, you need Druid version 24.0 or higher.
@@ -137,12 +134,12 @@ This configuration includes default settings that are appropriate for this tutor
the Druid root directory, apache-druid-24.0.0. Each service writes to a log file under <code>var/sv</code>.</p>
<p>At any time, you can revert Druid to its original, post-installation state by deleting the entire <code>var</code> directory. You may want to do this, for example, between Druid tutorials or after experimentation, to start with a fresh instance.</p>
<p>To stop Druid at any time, use CTRL+C in the terminal. This exits the <code>bin/start-micro-quickstart</code> script and terminates all Druid processes.</p>
-<h2><a class="anchor" aria-hidden="true" id="open-the-druid-console"></a><a href="#open-the-druid-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
-<p>After the Druid services finish startup, open the <a href="/docs/24.0.0/operations/druid-console.html">Druid console</a> at <a href="http://localhost:8888">http://localhost:8888</a>.</p>
-<p><img src="../assets/tutorial-quickstart-01.png" alt="Druid console" title="Druid console"></p>
-<p>It may take a few seconds for all Druid services to finish starting, including the <a href="/docs/24.0.0/design/router.html">Druid router</a>, which serves the console. If you attempt to open the Druid console before startup is complete, you may see errors in the browser. Wait a few moments and try again.</p>
-<p>In this quickstart, you use the the Druid console to perform ingestion. The MSQ task engine specifically uses the <strong>Query</strong> view to edit and run SQL queries.
-For a complete walkthrough of the <strong>Query</strong> view as it relates to the multi-stage query architecture and the MSQ task engine, see <a href="/docs/24.0.0/operations/druid-console.html">UI walkthrough</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="open-the-web-console"></a><a href="#open-the-web-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1 [...]
+<p>After the Druid services finish startup, open the <a href="/docs/24.0.0/operations/web-console.html">web console</a> at <a href="http://localhost:8888">http://localhost:8888</a>.</p>
+<p><img src="../assets/tutorial-quickstart-01.png" alt="web console" title="web console"></p>
+<p>It may take a few seconds for all Druid services to finish starting, including the <a href="/docs/24.0.0/design/router.html">Druid router</a>, which serves the console. If you attempt to open the web console before startup is complete, you may see errors in the browser. Wait a few moments and try again.</p>
+<p>In this quickstart, you use the the web console to perform ingestion. The MSQ task engine specifically uses the <strong>Query</strong> view to edit and run SQL queries.
+For a complete walkthrough of the <strong>Query</strong> view as it relates to the multi-stage query architecture and the MSQ task engine, see <a href="/docs/24.0.0/operations/web-console.html">UI walkthrough</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="load-data"></a><a href="#load-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
<p>The Druid distribution bundles the <code>wikiticker-2015-09-12-sampled.json.gz</code> sample dataset that you can use for testing. The sample dataset is located in the <code>quickstart/tutorial/</code> folder, accessible from the Druid root directory, and represents Wikipedia page edits for a given day.</p>
<p>Follow these steps to load the sample Wikipedia dataset:</p>
@@ -222,17 +219,15 @@ PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY<
<h2><a class="anchor" aria-hidden="true" id="next-steps"></a><a href="#next-steps" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<p>See the following topics for more information:</p>
<ul>
+<li><a href="/docs/24.0.0/querying/sql.html">Druid SQL overview</a> to learn about how to query the data you just ingested.</li>
+<li><a href="/docs/24.0.0/ingestion/index.html">Ingestion overview</a> to explore options for ingesting more data.</li>
+<li><a href="/docs/24.0.0/tutorials/tutorial-msq-extern.html">Tutorial: Load files using SQL</a> to learn how to generate a SQL query that loads external data into a Druid datasource.</li>
+<li><a href="/docs/24.0.0/tutorials/tutorial-batch-native.html">Tutorial: Load data with native batch ingestion</a> to load and query data with Druid's native batch ingestion feature.</li>
+<li><a href="/docs/24.0.0/tutorials/tutorial-kafka.html">Tutorial: Load stream data from Apache Kafka</a> to load streaming data from a Kafka topic.</li>
<li><a href="/docs/24.0.0/development/extensions.html">Extensions</a> for details on Druid extensions.</li>
-<li><a href="/docs/24.0.0/multi-stage-query/index.html#msq-task-engine-query-syntax">MSQ task engine query syntax</a> to further explore queries for SQL-based ingestion.</li>
-<li><a href="/docs/24.0.0/querying/sql.html">Druid SQL overview</a> to learn about how to query data you ingest.</li>
-<li><a href="/docs/24.0.0/tutorials/tutorial-batch-native.html">Load data with native batch ingestion</a> to load and query data with Druid's native batch ingestion feature.</li>
-<li><a href="/docs/24.0.0/tutorials/tutorial-kafka.html">Load stream data from Apache Kafka</a> to load streaming data from a Kafka topic.</li>
-<li><a href="/docs/24.0.0/multi-stage-query/api.html">API</a> to submit query tasks to the MSQ task engine programmatically.</li>
-<li><a href="/docs/24.0.0/multi-stage-query/connect-external-data.html">Connect external data</a> to learn how to generate a query that references externally hosted data that the MSQ task engine can use to ingest data.</li>
-<li><a href="/docs/24.0.0/multi-stage-query/convert-json-spec.html">Convert ingestion spec</a> to learn how to convert an existing JSON ingestion spec to a SQL query that the MSQ task engine can use to ingest data.</li>
</ul>
-<p>Remember that after stopping Druid services, you can start clean next time by deleting the <code>var</code> directory from the Druid root directory and running the <code>bin/start-micro-quickstart</code> script again. You may want to do this before taking other data ingestion tutorials, since they use the same Wikipedia datasource.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/design/index.html"><span class="arrow-prev">← </span><span>Introduction to Apache Druid</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/docker.html"><span>Docker</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#requirements">Requirements</a></li><li><a href="#install-druid">Install Druid</a></l [...]
+<p>Remember that after stopping Druid services, you can start clean next time by deleting the <code>var</code> directory from the Druid root directory and running the <code>bin/start-micro-quickstart</code> script again. You may want to do this before using other data ingestion tutorials, since they use the same Wikipedia datasource.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/design/index.html"><span class="arrow-prev">← </span><span>Introduction to Apache Druid</span></a><a class="docs-next button" href="/docs/24.0.0/operations/single-server.html"><span>Single server deployment</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#requirements">Requirements</a></li><li><a href="#install-d [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-batch-hadoop.html b/docs/24.0.0/tutorials/tutorial-batch-hadoop.html
index 59a92f33..9ddf8b44 100644
--- a/docs/24.0.0/tutorials/tutorial-batch-hadoop.html
+++ b/docs/24.0.0/tutorials/tutorial-batch-hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/tutorials/tutorial-batch-native.html b/docs/24.0.0/tutorials/tutorial-batch-native.html
index 465aca89..a38a0e19 100644
--- a/docs/24.0.0/tutorials/tutorial-batch-native.html
+++ b/docs/24.0.0/tutorials/tutorial-batch-native.html
@@ -23,14 +23,14 @@
-->
<p>This topic shows you how to load and query data files in Apache Druid using its native batch ingestion feature.</p>
<h2><a class="anchor" aria-hidden="true" id="prerequisites"></a><a href="#prerequisites" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.8 [...]
-<p>Install Druid, start up Druid services, and open the Druid console as described in the <a href="/docs/24.0.0/tutorials/index.html">Druid quickstart</a>.</p>
+<p>Install Druid, start up Druid services, and open the web console as described in the <a href="/docs/24.0.0/tutorials/index.html">Druid quickstart</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="load-data"></a><a href="#load-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
<p>Ingestion specs define the schema of the data Druid reads and stores. You can write ingestion specs by hand or using the <em>data loader</em>,
as we'll do here to perform batch file loading with Druid's native batch ingestion.</p>
<p>The Druid distribution bundles sample data we can use. The sample data located in <code>quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz</code>
in the Druid root directory represents Wikipedia page edits for a given day.</p>
<ol>
-<li><p>Click <strong>Load data</strong> from the Druid console header (<img src="../assets/tutorial-batch-data-loader-00.png" alt="Load data">).</p></li>
+<li><p>Click <strong>Load data</strong> from the web console header (<img src="../assets/tutorial-batch-data-loader-00.png" alt="Load data">).</p></li>
<li><p>Select the <strong>Local disk</strong> tile and then click <strong>Connect data</strong>.</p>
<p><img src="../assets/tutorial-batch-data-loader-01.png" alt="Data loader init" title="Data loader init"></p></li>
<li><p>Enter the following values:</p>
diff --git a/docs/24.0.0/tutorials/tutorial-batch.html b/docs/24.0.0/tutorials/tutorial-batch.html
index d9349435..8cfdeb9d 100644
--- a/docs/24.0.0/tutorials/tutorial-batch.html
+++ b/docs/24.0.0/tutorials/tutorial-batch.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -79,11 +79,10 @@
-->
<p>This tutorial demonstrates how to load data into Apache Druid from a file using Apache Druid's native batch ingestion feature.</p>
<p>You initiate data loading in Druid by submitting an <em>ingestion task</em> spec to the Druid Overlord. You can write ingestion
-specs by hand or using the <em>data loader</em> built into the Druid console.</p>
-<p>The <a href="/docs/24.0.0/tutorials/index.html">Quickstart</a> shows you how to use the data loader to build an ingestion spec. For production environments, it's
-likely that you'll want to automate data ingestion. This tutorial starts by showing you how to submit an ingestion spec
-directly in the Druid console, and then introduces ways to ingest batch data that lend themselves to
-automation—from the command line and from a script.</p>
+specs by hand or using the <em>data loader</em> built into the web console.</p>
+<p>For production environments, it's likely that you'll want to automate data ingestion. This tutorial starts by showing
+you how to submit an ingestion spec directly in the web console, and then introduces ways to ingest batch data that
+lend themselves to automation—from the command line and from a script.</p>
<h2><a class="anchor" aria-hidden="true" id="loading-data-with-a-spec-via-console"></a><a href="#loading-data-with-a-spec-via-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 [...]
<p>The Druid package includes the following sample native batch ingestion task spec at <code>quickstart/tutorial/wikipedia-index.json</code>, shown here for convenience,
which has been configured to read the <code>quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz</code> input file:</p>
@@ -189,7 +188,7 @@ wikipedia loading complete! You may now query your data
<p>If you wish to go through any of the other ingestion tutorials, you will need to shut down the cluster and reset the cluster state by removing the contents of the <code>var</code> directory under the druid package, as the other tutorials will write to the same "wikipedia" datasource.</p>
<h2><a class="anchor" aria-hidden="true" id="further-reading"></a><a href="#further-reading" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
<p>For more information on loading batch data, please see <a href="/docs/24.0.0/ingestion/native-batch.html">the native batch ingestion documentation</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/cluster.html"><span class="arrow-prev">← </span><span>Clustered deployment</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-msq-external-data.html"><span>Loading files with SQL</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#loading-data-with-a-spec-via-console">Loading data [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/cluster.html"><span class="arrow-prev">← </span><span>Clustered deployment</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-msq-extern.html"><span>Load files using SQL 🆕</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#loading-data-with-a-spec-via-console">Loading data with a [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-compaction.html b/docs/24.0.0/tutorials/tutorial-compaction.html
index 7d4434ee..e7901c88 100644
--- a/docs/24.0.0/tutorials/tutorial-compaction.html
+++ b/docs/24.0.0/tutorials/tutorial-compaction.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -99,7 +99,7 @@ Submit the spec as follows to create a datasource called <code>compaction-tutori
<blockquote>
<p><code>maxRowsPerSegment</code> in the tutorial ingestion spec is set to 1000 to generate multiple segments per hour for demonstration purposes. Do not use this spec in production.</p>
</blockquote>
-<p>After the ingestion completes, navigate to <a href="http://localhost:8888/unified-console.html#datasources">http://localhost:8888/unified-console.html#datasources</a> in a browser to see the new datasource in the Druid console.</p>
+<p>After the ingestion completes, navigate to <a href="http://localhost:8888/unified-console.html#datasources">http://localhost:8888/unified-console.html#datasources</a> in a browser to see the new datasource in the web console.</p>
<p><img src="../assets/tutorial-compaction-01.png" alt="compaction-tutorial datasource" title="compaction-tutorial datasource"></p>
<p>In the <strong>Availability</strong> column for the <code>compaction-tutorial</code> datasource, click the link for <strong>51 segments</strong> to view segments information for the datasource.</p>
<p>The datasource comprises 51 segments, between one and three segments per hour from the input data:</p>
@@ -142,7 +142,7 @@ During that time, you may see 75 total segments comprised of the old segment set
<p><img src="../assets/tutorial-compaction-03.png" alt="Compacted segments intermediate state 1" title="Compacted segments intermediate state 1"></p>
<p><img src="../assets/tutorial-compaction-04.png" alt="Compacted segments intermediate state 2" title="Compacted segments intermediate state 2"></p>
<p>The new compacted segments have a more recent version than the original segments.
-Even though the Druid console displays both sets of segments, queries only read from the new compacted segments.</p>
+Even though the web console displays both sets of segments, queries only read from the new compacted segments.</p>
<p>Run a COUNT query on <code>compaction-tutorial</code> again to verify the number of rows remains 39,244:</p>
<pre><code class="hljs css language-bash">dsql> select count(*) from <span class="hljs-string">"compaction-tutorial"</span>;
┌────────┐
@@ -186,7 +186,7 @@ Retrieved 1 row <span class="hljs-keyword">in</span> 1.30s.
<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
<p>This tutorial demonstrated how to use a compaction task spec to manually compact segments and how to optionally change the segment granularity for segments.</p>
<ul>
-<li>For more details, see <a href="/docs/24.0.0/ingestion/compaction.html">Compaction</a>.</li>
+<li>For more details, see <a href="/docs/24.0.0/data-management/compaction.html">Compaction</a>.</li>
<li>To learn about the benefits of compaction, see <a href="/docs/24.0.0/operations/segment-optimization.html">Segment optimization</a>.</li>
</ul>
</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-update-data.html"><span class="arrow-prev">← </span><span>Updating existing data</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-delete-data.html"><span>Deleting data</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#prerequisites">Prerequisites</a></li><li><a href="# [...]
diff --git a/docs/24.0.0/tutorials/tutorial-delete-data.html b/docs/24.0.0/tutorials/tutorial-delete-data.html
index a8143ffa..b3a68966 100644
--- a/docs/24.0.0/tutorials/tutorial-delete-data.html
+++ b/docs/24.0.0/tutorials/tutorial-delete-data.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/tutorials/tutorial-ingestion-spec.html b/docs/24.0.0/tutorials/tutorial-ingestion-spec.html
index 08706d5d..22aff6e8 100644
--- a/docs/24.0.0/tutorials/tutorial-ingestion-spec.html
+++ b/docs/24.0.0/tutorials/tutorial-ingestion-spec.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/tutorials/tutorial-kafka.html b/docs/24.0.0/tutorials/tutorial-kafka.html
index 702f7013..66c5636a 100644
--- a/docs/24.0.0/tutorials/tutorial-kafka.html
+++ b/docs/24.0.0/tutorials/tutorial-kafka.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -230,7 +230,7 @@ Similarly, you can also edit the spec directly and see it reflected in the previ
<p>If the supervisor was successfully created, you will get a response containing the ID of the supervisor; in our case we should see <code>{"id":"wikipedia"}</code>.</p>
<p>For more details about what's going on here, check out the
<a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Druid Kafka indexing service documentation</a>.</p>
-<p>You can view the current supervisors and tasks in the Druid console: <a href="http://localhost:8888/unified-console.html#tasks">http://localhost:8888/unified-console.md#tasks</a>.</p>
+<p>You can view the current supervisors and tasks in the web console: <a href="http://localhost:8888/unified-console.html#tasks">http://localhost:8888/unified-console.md#tasks</a>.</p>
<h2><a class="anchor" aria-hidden="true" id="querying-your-data"></a><a href="#querying-your-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22- [...]
<p>After data is sent to the Kafka stream, it is immediately available for querying.</p>
<p>Please follow the <a href="/docs/24.0.0/tutorials/tutorial-query.html">query tutorial</a> to run some example queries on the newly loaded data.</p>
@@ -241,7 +241,7 @@ Similarly, you can also edit the spec directly and see it reflected in the previ
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="further-reading"></a><a href="#further-reading" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
<p>For more information on loading data from Kafka streams, please see the <a href="/docs/24.0.0/development/extensions-core/kafka-ingestion.html">Druid Kafka indexing service documentation</a>.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-msq-external-data.html"><span class="arrow-prev">← </span><span>Loading files with SQL</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-batch-hadoop.html"><span>Load from Apache Hadoop</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#getting-started">Getting started</ [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-msq-extern.html"><span class="arrow-prev">← </span><span>Load files using SQL 🆕</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-batch-hadoop.html"><span>Load from Apache Hadoop</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#getting-started">Getting started</a></li> [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html b/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html
index 527c234b..899ca07d 100644
--- a/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html
+++ b/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -148,7 +148,7 @@ For example, if druid processes run as user 'root', then</p>
</code></pre>
<h3><a class="anchor" aria-hidden="true" id="restart-druid-services"></a><a href="#restart-druid-services" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
<p>With the above changes, restart Druid. This would ensure that Druid works with Kerberized Hadoop</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-transform-spec.html"><span class="arrow-prev">← </span><span>Transforming input data</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-msq-convert-json.html"><span>Convert ingestion spec to SQL</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#hadoop-setup">Hadoop Setup [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/docker.html"><span class="arrow-prev">← </span><span>Tutorial: Run with Docker</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html"><span>Convert ingestion spec to SQL</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#hadoop-setup">Hadoop Setup</a><ul class=" [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-msq-convert-json.html b/docs/24.0.0/tutorials/tutorial-msq-convert-json.html
deleted file mode 100644
index 4f5de0dc..00000000
--- a/docs/24.0.0/tutorials/tutorial-msq-convert-json.html
+++ /dev/null
@@ -1,117 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Convert ingestion spec to SQL · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/tutorial-msq-convert-json.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:ve [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/tutorial-msq-convert-json.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Convert ingestion spec to SQL</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<!DOCTYPE html>
-<!--This redirects to the Multi-Stage Query tutorial. This redirect file exists cause duplicate entries in the left nav aren't allowed-->
-<html lang="en-US">
- <head>
- <meta charset="UTF-8" />
- <meta
- http-equiv="refresh"
- content="0; url=/docs/multi-stage-query/convert-json-spec.html"
- />
- <script type="text/javascript">
- window.location.href = '/docs/multi-stage-query/convert-json-spec.html';
- </script>
- <title>About the Druid documentation</title>
- </head>
- <body>
- If you are not redirected automatically, follow this
- <a href="/docs/24.0.0/multi-stage-query/convert-json-spec.html">link</a>.
- </body>
-</html></span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html"><span class="arrow-prev">← </span><span>Kerberized HDFS deep storage</span></a><a class="docs-next button" href="/docs/24.0.0/design/architecture.html"><span>Design</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html b/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html
new file mode 100644
index 00000000..b4dac7bc
--- /dev/null
+++ b/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html
@@ -0,0 +1,235 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial: Convert an ingestion spec for SQL-based ingestion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/tutorial-msq-convert-spec.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="How to convert an ingestion spec to a query for SQ [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/tutorial-msq-convert-spec.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Tutorial: Convert an ingestion spec for SQL-based ingestion</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<blockquote>
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
+</blockquote>
+<p>If you're already ingesting data with <a href="/docs/24.0.0/ingestion/native-batch.html">native batch ingestion</a>, you can use the <a href="/docs/24.0.0/operations/web-console.html">web console</a> to convert the ingestion spec to a SQL query that the multi-stage query task engine can use to ingest data.</p>
+<p>This tutorial demonstrates how to convert the ingestion spec to a query task in the web console.</p>
+<p>To convert the ingestion spec to a query task, do the following:</p>
+<ol>
+<li><p>In the <strong>Query</strong> view of the web console, navigate to the menu bar that includes <strong>Run</strong>.</p></li>
+<li><p>Click the ellipsis icon and select <strong>Convert ingestion spec to SQL</strong>.
+<img src="../assets/multi-stage-query/tutorial-msq-convert.png" alt="Convert ingestion spec to SQL" title="Convert ingestion spec to SQL"></p></li>
+<li><p>In the <strong>Ingestion spec to covert</strong> window, insert your ingestion spec. You can use your own spec or the sample ingestion spec provided in the tutorial. The sample spec uses data hosted at <code>https://druid.apache.org/data/wikipedia.json.gz</code> and loads it into a table named <code>wikipedia</code>:</p>
+<p><details><summary>Show the spec</summary></p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
+ <span class="hljs-attr">"spec"</span>: {
+ <span class="hljs-attr">"ioConfig"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
+ <span class="hljs-attr">"inputSource"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"http"</span>,
+ <span class="hljs-attr">"uris"</span>: [
+ <span class="hljs-string">"https://druid.apache.org/data/wikipedia.json.gz"</span>
+ ]
+ },
+ <span class="hljs-attr">"inputFormat"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"json"</span>
+ }
+ },
+ <span class="hljs-attr">"tuningConfig"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"index_parallel"</span>,
+ <span class="hljs-attr">"partitionsSpec"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"dynamic"</span>
+ }
+ },
+ <span class="hljs-attr">"dataSchema"</span>: {
+ <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
+ <span class="hljs-attr">"timestampSpec"</span>: {
+ <span class="hljs-attr">"column"</span>: <span class="hljs-string">"timestamp"</span>,
+ <span class="hljs-attr">"format"</span>: <span class="hljs-string">"iso"</span>
+ },
+ <span class="hljs-attr">"dimensionsSpec"</span>: {
+ <span class="hljs-attr">"dimensions"</span>: [
+ <span class="hljs-string">"isRobot"</span>,
+ <span class="hljs-string">"channel"</span>,
+ <span class="hljs-string">"flags"</span>,
+ <span class="hljs-string">"isUnpatrolled"</span>,
+ <span class="hljs-string">"page"</span>,
+ <span class="hljs-string">"diffUrl"</span>,
+ {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
+ <span class="hljs-attr">"name"</span>: <span class="hljs-string">"added"</span>
+ },
+ <span class="hljs-string">"comment"</span>,
+ {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
+ <span class="hljs-attr">"name"</span>: <span class="hljs-string">"commentLength"</span>
+ },
+ <span class="hljs-string">"isNew"</span>,
+ <span class="hljs-string">"isMinor"</span>,
+ {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
+ <span class="hljs-attr">"name"</span>: <span class="hljs-string">"delta"</span>
+ },
+ <span class="hljs-string">"isAnonymous"</span>,
+ <span class="hljs-string">"user"</span>,
+ {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
+ <span class="hljs-attr">"name"</span>: <span class="hljs-string">"deltaBucket"</span>
+ },
+ {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"long"</span>,
+ <span class="hljs-attr">"name"</span>: <span class="hljs-string">"deleted"</span>
+ },
+ <span class="hljs-string">"namespace"</span>,
+ <span class="hljs-string">"cityName"</span>,
+ <span class="hljs-string">"countryName"</span>,
+ <span class="hljs-string">"regionIsoCode"</span>,
+ <span class="hljs-string">"metroCode"</span>,
+ <span class="hljs-string">"countryIsoCode"</span>,
+ <span class="hljs-string">"regionName"</span>
+ ]
+ },
+ <span class="hljs-attr">"granularitySpec"</span>: {
+ <span class="hljs-attr">"queryGranularity"</span>: <span class="hljs-string">"none"</span>,
+ <span class="hljs-attr">"rollup"</span>: <span class="hljs-literal">false</span>,
+ <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"day"</span>
+ }
+ }
+ }
+}
+</code></pre>
+<p></details></p></li>
+<li><p>Click <strong>Submit</strong> to submit the spec. The web console uses the JSON-based ingestion spec to generate a SQL query that you can use instead. This is what the query looks like for the sample ingestion spec:</p>
+<p><details><summary>Show the query</summary></p>
+<pre><code class="hljs css language-sql"><span class="hljs-comment">-- This SQL query was auto generated from an ingestion spec</span>
+<span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> wikipedia OVERWRITE <span class="hljs-keyword">ALL</span>
+<span class="hljs-keyword">WITH</span> <span class="hljs-keyword">source</span> <span class="hljs-keyword">AS</span> (<span class="hljs-keyword">SELECT</span> * <span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
+ EXTERN(
+ <span class="hljs-string">'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
+ <span class="hljs-string">'{"type":"json"}'</span>,
+ <span class="hljs-string">'[{"name":"timestamp","type":"string"},{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name": [...]
+ )
+))
+<span class="hljs-keyword">SELECT</span>
+ TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
+ <span class="hljs-string">"isRobot"</span>,
+ <span class="hljs-string">"channel"</span>,
+ <span class="hljs-string">"flags"</span>,
+ <span class="hljs-string">"isUnpatrolled"</span>,
+ <span class="hljs-string">"page"</span>,
+ <span class="hljs-string">"diffUrl"</span>,
+ <span class="hljs-string">"added"</span>,
+ <span class="hljs-string">"comment"</span>,
+ <span class="hljs-string">"commentLength"</span>,
+ <span class="hljs-string">"isNew"</span>,
+ <span class="hljs-string">"isMinor"</span>,
+ <span class="hljs-string">"delta"</span>,
+ <span class="hljs-string">"isAnonymous"</span>,
+ <span class="hljs-string">"user"</span>,
+ <span class="hljs-string">"deltaBucket"</span>,
+ <span class="hljs-string">"deleted"</span>,
+ <span class="hljs-string">"namespace"</span>,
+ <span class="hljs-string">"cityName"</span>,
+ <span class="hljs-string">"countryName"</span>,
+ <span class="hljs-string">"regionIsoCode"</span>,
+ <span class="hljs-string">"metroCode"</span>,
+ <span class="hljs-string">"countryIsoCode"</span>,
+ <span class="hljs-string">"regionName"</span>
+<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">source</span>
+PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
+</code></pre>
+<p></details></p></li>
+<li><p>Review the generated SQL query to make sure it matches your requirements and does what you expect.</p></li>
+<li><p>Click <strong>Run</strong> to start the ingestion.</p></li>
+</ol>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html"><span class="arrow-prev">← </span><span>Kerberized HDFS deep storage</span></a><a class="docs-next button" href="/docs/24.0.0/design/architecture.html"><span>Design</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class= [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/tutorials/tutorial-msq-extern.html b/docs/24.0.0/tutorials/tutorial-msq-extern.html
new file mode 100644
index 00000000..83249036
--- /dev/null
+++ b/docs/24.0.0/tutorials/tutorial-msq-extern.html
@@ -0,0 +1,203 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Tutorial: Load files with SQL-based ingestion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/tutorial-msq-extern.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="How to generate a query that references externally hosted data"/><meta [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/tutorial-msq-extern.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Tutorial: Load files with SQL-based ingestion</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<blockquote>
+<p>This page describes SQL-based batch ingestion using the <a href="/docs/24.0.0/multi-stage-query/index.html"><code>druid-multi-stage-query</code></a>
+extension, new in Druid 24.0. Refer to the <a href="/docs/24.0.0/ingestion/index.html#batch">ingestion methods</a> table to determine which
+ingestion method is right for you.</p>
+</blockquote>
+<p>This tutorial demonstrates how to generate a query that references externally hosted data using the <strong>Connect external data</strong> wizard.</p>
+<p>The following example uses EXTERN to query a JSON file located at <a href="https://druid.apache.org/data/wikipedia.json.gz">https://druid.apache.org/data/wikipedia.json.gz</a>.</p>
+<p>Although you can manually create a query in the UI, you can use Druid to generate a base query for you that you can modify to meet your requirements.</p>
+<p>To generate a query from external data, do the following:</p>
+<ol>
+<li><p>In the <strong>Query</strong> view of the web console, click <strong>Connect external data</strong>.</p></li>
+<li><p>On the <strong>Select input type</strong> screen, choose <strong>HTTP(s)</strong> and enter the following value in the <strong>URIs</strong> field: <code>https://druid.apache.org/data/wikipedia.json.gz</code>. Leave the HTTP auth username and password blank.</p></li>
+<li><p>Click <strong>Connect data</strong>.</p></li>
+<li><p>On the <strong>Parse</strong> screen, you can perform additional actions before you load the data into Druid:</p>
+<ul>
+<li>Expand a row to see what data it corresponds to from the source.</li>
+<li>Customize how Druid handles the data by selecting the <strong>Input format</strong> and its related options, such as adding <strong>JSON parser features</strong> for JSON files.</li>
+</ul></li>
+<li><p>When you're ready, click <strong>Done</strong>. You're returned to the <strong>Query</strong> view where you can see the starter query that will insert the data from the external source into a table named <code>wikipedia</code>.</p>
+<p><details><summary>Show the query</summary></p>
+<pre><code class="hljs css language-sql"><span class="hljs-keyword">REPLACE</span> <span class="hljs-keyword">INTO</span> <span class="hljs-string">"wikipedia"</span> OVERWRITE <span class="hljs-keyword">ALL</span>
+<span class="hljs-keyword">WITH</span> ext <span class="hljs-keyword">AS</span> (<span class="hljs-keyword">SELECT</span> *
+<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
+ EXTERN(
+ <span class="hljs-string">'{"type":"http","uris":["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
+ <span class="hljs-string">'{"type":"json"}'</span>,
+ <span class="hljs-string">'[{"name":"isRobot","type":"string"},{"name":"channel","type":"string"},{"name":"timestamp","type":"string"},{"name":"flags","type":"string"},{"name":"isUnpatrolled","type":"string"},{"name":"page","type":"string"},{"name":"diffUrl","type":"string"},{"name":"added","type":"long"},{"name":"comment","type":"string"},{"name":"commentLength","type":"long"},{"name":"isNew","type":"string"},{"name":"isMinor","type":"string"},{"name":"delta","type":"long"},{"name": [...]
+ )
+))
+<span class="hljs-keyword">SELECT</span>
+ TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
+ isRobot,
+ channel,
+ flags,
+ isUnpatrolled,
+ page,
+ diffUrl,
+ added,
+ <span class="hljs-keyword">comment</span>,
+ commentLength,
+ isNew,
+ isMinor,
+ delta,
+ isAnonymous,
+ <span class="hljs-keyword">user</span>,
+ deltaBucket,
+ deleted,
+ namespace,
+ cityName,
+ countryName,
+ regionIsoCode,
+ metroCode,
+ countryIsoCode,
+ regionName
+<span class="hljs-keyword">FROM</span> ext
+PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
+</code></pre>
+<p></details></p></li>
+<li><p>Review and modify the query to meet your needs. For example, you can rename the table or change segment granularity. To partition by something other than ALL, include <code>TIME_PARSE("timestamp") AS __time</code> in your SELECT statement.</p>
+<p>For example, to specify day-based segment granularity, change the partitioning to <code>PARTITIONED BY DAY</code>:</p>
+<pre><code class="hljs css language-sql"> <span class="hljs-keyword">INSERT</span> <span class="hljs-keyword">INTO</span> ...
+ <span class="hljs-keyword">SELECT</span>
+ TIME_PARSE(<span class="hljs-string">"timestamp"</span>) <span class="hljs-keyword">AS</span> __time,
+ ...
+ ...
+ PARTITIONED <span class="hljs-keyword">BY</span> <span class="hljs-keyword">DAY</span>
+</code></pre></li>
+<li><p>Optionally, select <strong>Preview</strong> to review the data before you ingest it. A preview runs the query without the REPLACE INTO clause and with an added LIMIT.
+You can see the general shape of the data before you commit to inserting it.
+The LIMITs make the query run faster but can cause incomplete results.</p></li>
+<li><p>Click <strong>Run</strong> to launch your query. The query returns information including its duration and the number of rows inserted into the table.</p></li>
+</ol>
+<h2><a class="anchor" aria-hidden="true" id="query-the-data"></a><a href="#query-the-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0- [...]
+<p>You can query the <code>wikipedia</code> table after the ingestion completes.
+For example, you can analyze the data in the table to produce a list of top channels:</p>
+<pre><code class="hljs css language-sql"><span class="hljs-keyword">SELECT</span>
+ channel,
+ <span class="hljs-keyword">COUNT</span>(*)
+<span class="hljs-keyword">FROM</span> <span class="hljs-string">"wikipedia"</span>
+<span class="hljs-keyword">GROUP</span> <span class="hljs-keyword">BY</span> channel
+<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">COUNT</span>(*) <span class="hljs-keyword">DESC</span>
+</code></pre>
+<p>With the EXTERN function, you could run the same query on the external data directly without ingesting it first:</p>
+<p><details><summary>Show the query</summary></p>
+<pre><code class="hljs css language-sql"><span class="hljs-keyword">SELECT</span>
+ channel,
+ <span class="hljs-keyword">COUNT</span>(*)
+<span class="hljs-keyword">FROM</span> <span class="hljs-keyword">TABLE</span>(
+ EXTERN(
+ <span class="hljs-string">'{"type": "http", "uris": ["https://druid.apache.org/data/wikipedia.json.gz"]}'</span>,
+ <span class="hljs-string">'{"type": "json"}'</span>,
+ <span class="hljs-string">'[{"name": "added", "type": "long"}, {"name": "channel", "type": "string"}, {"name": "cityName", "type": "string"}, {"name": "comment", "type": "string"}, {"name": "commentLength", "type": "long"}, {"name": "countryIsoCode", "type": "string"}, {"name": "countryName", "type": "string"}, {"name": "deleted", "type": "long"}, {"name": "delta", "type": "long"}, {"name": "deltaBucket", "type": "string"}, {"name": "diffUrl", "type": "string"}, {"name": "flags", "ty [...]
+ )
+)
+<span class="hljs-keyword">GROUP</span> <span class="hljs-keyword">BY</span> channel
+<span class="hljs-keyword">ORDER</span> <span class="hljs-keyword">BY</span> <span class="hljs-keyword">COUNT</span>(*) <span class="hljs-keyword">DESC</span>
+</code></pre>
+<p></details></p>
+<h2><a class="anchor" aria-hidden="true" id="further-reading"></a><a href="#further-reading" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
+<p>See the following topics to learn more:</p>
+<ul>
+<li><a href="/docs/24.0.0/multi-stage-query/index.html">SQL-based ingestion overview</a> to further explore SQL-based ingestion.</li>
+<li><a href="/docs/24.0.0/multi-stage-query/reference.html">SQL-based ingestion reference</a> for reference on context parameters, functions, and error codes.</li>
+</ul>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-batch.html"><span class="arrow-prev">← </span><span>Load files natively</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-kafka.html"><span>Load from Apache Kafka</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-the-data">Query the data</a></li><li><a href="#furt [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/tutorials/tutorial-msq-external-data.html b/docs/24.0.0/tutorials/tutorial-msq-external-data.html
deleted file mode 100644
index d76c126e..00000000
--- a/docs/24.0.0/tutorials/tutorial-msq-external-data.html
+++ /dev/null
@@ -1,117 +0,0 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Loading files with SQL · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/24.0.0/tutorials/tutorial-msq-external-data.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" [...]
- window.dataLayer = window.dataLayer || [];
- function gtag(){dataLayer.push(arguments); }
- gtag('js', new Date());
- gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
- var coll = document.getElementsByClassName('collapsible');
- var checkActiveCategory = true;
- for (var i = 0; i < coll.length; i++) {
- var links = coll[i].nextElementSibling.getElementsByTagName('*');
- if (checkActiveCategory){
- for (var j = 0; j < links.length; j++) {
- if (links[j].classList.contains('navListItemActive')){
- coll[i].nextElementSibling.classList.toggle('hide');
- coll[i].childNodes[1].classList.toggle('rotate');
- checkActiveCategory = false;
- break;
- }
- }
- }
-
- coll[i].addEventListener('click', function() {
- var arrow = this.childNodes[1];
- arrow.classList.toggle('rotate');
- var content = this.nextElementSibling;
- content.classList.toggle('hide');
- });
- }
-
- document.addEventListener('DOMContentLoaded', function() {
- createToggler('#navToggler', '#docsNav', 'docsSliderActive');
- createToggler('#tocToggler', 'body', 'tocActive');
-
- var headings = document.querySelector('.toc-headings');
- headings && headings.addEventListener('click', function(event) {
- var el = event.target;
- while(el !== headings){
- if (el.tagName === 'A') {
- document.body.classList.remove('tocActive');
- break;
- } else{
- el = el.parentNode;
- }
- }
- }, false);
-
- function createToggler(togglerSelector, targetSelector, className) {
- var toggler = document.querySelector(togglerSelector);
- var target = document.querySelector(targetSelector);
-
- if (!toggler) {
- return;
- }
-
- toggler.onclick = function(event) {
- event.preventDefault();
-
- target.classList.toggle(className);
- };
- }
- });
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/tutorials/tutorial-msq-connect-extern.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Loading files with SQL</h1></header><article><div><span><!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-<!DOCTYPE html>
-<!--This redirects to the Multi-Stage Query tutorial. This redirect file exists cause duplicate entries in the left nav aren't allowed-->
-<html lang="en-US">
- <head>
- <meta charset="UTF-8" />
- <meta
- http-equiv="refresh"
- content="0; url=/docs/multi-stage-query/connect-external-data.html"
- />
- <script type="text/javascript">
- window.location.href = '/docs/multi-stage-query/connect-external-data.html';
- </script>
- <title>About the Druid documentation</title>
- </head>
- <body>
- If you are not redirected automatically, follow this
- <a href="/docs/24.0.0/multi-stage-query/connect-external-data.html">link</a>.
- </body>
-</html></span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-batch.html"><span class="arrow-prev">← </span><span>Loading files natively</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-kafka.html"><span>Load from Apache Kafka</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container" [...]
- document.addEventListener('keyup', function(e) {
- if (e.target !== document.body) {
- return;
- }
- // keyCode for '/' (slash)
- if (e.keyCode === 191) {
- const search = document.getElementById('search_input_react');
- search && search.focus();
- }
- });
- </script><script>
- var search = docsearch({
-
- apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
- indexName: 'apache_druid',
- inputSelector: '#search_input_react',
- algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
- });
- </script></body></html>
\ No newline at end of file
diff --git a/docs/24.0.0/tutorials/tutorial-query.html b/docs/24.0.0/tutorials/tutorial-query.html
index 54d81452..1c330579 100644
--- a/docs/24.0.0/tutorials/tutorial-query.html
+++ b/docs/24.0.0/tutorials/tutorial-query.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -86,13 +86,13 @@ by following one of them:</p>
<li><a href="/docs/24.0.0/tutorials/tutorial-kafka.html">Tutorial: Loading stream data from Kafka</a></li>
<li><a href="/docs/24.0.0/tutorials/tutorial-batch-hadoop.html">Tutorial: Loading a file using Hadoop</a></li>
</ul>
-<p>There are various ways to run Druid SQL queries: from the Druid console, using a command line utility
+<p>There are various ways to run Druid SQL queries: from the web console, using a command line utility
and by posting the query by HTTP. We'll look at each of these.</p>
-<h2><a class="anchor" aria-hidden="true" id="query-sql-from-the-druid-console"></a><a href="#query-sql-from-the-druid-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13. [...]
-<p>The Druid console includes a view that makes it easier to build and test queries, and
+<h2><a class="anchor" aria-hidden="true" id="query-sql-from-the-web-console"></a><a href="#query-sql-from-the-web-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>The web console includes a view that makes it easier to build and test queries, and
view their results.</p>
<ol>
-<li><p>Start up the Druid cluster, if it's not already running, and open the Druid console in your web
+<li><p>Start up the Druid cluster, if it's not already running, and open the web console in your web
browser.</p></li>
<li><p>Click <strong>Query</strong> from the header to open the Query view:</p>
<p><img src="../assets/tutorial-query-01.png" alt="Query view" title="Query view"></p>
@@ -168,7 +168,7 @@ from the command line or over HTTP.</p>
<ol start="9">
<li>Finally, click <code>...</code> and <strong>Edit context</strong> to see how you can add additional parameters controlling the execution of the query execution. In the field, enter query context options as JSON key-value pairs, as described in <a href="/docs/24.0.0/querying/query-context.html">Context flags</a>.</li>
</ol>
-<p>That's it! We've built a simple query using some of the query builder features built into the Druid console. The following
+<p>That's it! We've built a simple query using some of the query builder features built into the web console. The following
sections provide a few more example queries you can try. Also, see <a href="#other-ways-to-invoke-sql-queries">Other ways to invoke SQL queries</a> to learn how
to run Druid SQL from the command line or over HTTP.</p>
<h2><a class="anchor" aria-hidden="true" id="more-druid-sql-examples"></a><a href="#more-druid-sql-examples" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 [...]
@@ -268,7 +268,7 @@ Retrieved 10 rows in 0.06s.
<h2><a class="anchor" aria-hidden="true" id="further-reading"></a><a href="#further-reading" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 [...]
<p>See the <a href="/docs/24.0.0/querying/sql.html">Druid SQL documentation</a> for more information on using Druid SQL queries.</p>
<p>See the <a href="/docs/24.0.0/querying/querying.html">Queries documentation</a> for more information on Druid native queries.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-batch-hadoop.html"><span class="arrow-prev">← </span><span>Load from Apache Hadoop</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-rollup.html"><span>Roll-up</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-sql-from-the-druid-console">Query SQL from the Druid c [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-batch-hadoop.html"><span class="arrow-prev">← </span><span>Load from Apache Hadoop</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-rollup.html"><span>Roll-up</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#query-sql-from-the-web-console">Query SQL from the web conso [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-retention.html b/docs/24.0.0/tutorials/tutorial-retention.html
index 543f23c7..a6962e02 100644
--- a/docs/24.0.0/tutorials/tutorial-retention.html
+++ b/docs/24.0.0/tutorials/tutorial-retention.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -86,7 +86,7 @@ the <a href="/docs/24.0.0/tutorials/index.html">single-machine quickstart</a> an
<p>The ingestion spec can be found at <code>quickstart/tutorial/retention-index.json</code>. Let's submit that spec, which will create a datasource called <code>retention-tutorial</code>:</p>
<pre><code class="hljs css language-bash">bin/post-index-task --file quickstart/tutorial/retention-index.json --url http://localhost:8081
</code></pre>
-<p>After the ingestion completes, go to <a href="http://localhost:8888/unified-console.html#datasources">http://localhost:8888/unified-console.html#datasources</a> in a browser to access the Druid console's datasource view.</p>
+<p>After the ingestion completes, go to <a href="http://localhost:8888/unified-console.html#datasources">http://localhost:8888/unified-console.html#datasources</a> in a browser to access the web console's datasource view.</p>
<p>This view shows the available datasources and a summary of the retention rules for each datasource:</p>
<p><img src="../assets/tutorial-retention-01.png" alt="Summary" title="Summary"></p>
<p>Currently there are no rules set for the <code>retention-tutorial</code> datasource. Note that there are default rules for the cluster: load forever with 2 replicas in <code>_default_tier</code>.</p>
@@ -108,7 +108,7 @@ the <a href="/docs/24.0.0/tutorials/index.html">single-machine quickstart</a> an
<p>Now click <code>Next</code>. The rule configuration process will ask for a user name and comment, for change logging purposes. You can enter <code>tutorial</code> for both.</p>
<p>Now click <code>Save</code>. You can see the new rules in the datasources view:</p>
<p><img src="../assets/tutorial-retention-05.png" alt="New rules" title="New rules"></p>
-<p>Give the cluster a few minutes to apply the rule change, and go to the <a href="http://localhost:8888/unified-console.html#segments">segments view</a> in the Druid console.
+<p>Give the cluster a few minutes to apply the rule change, and go to the <a href="http://localhost:8888/unified-console.html#segments">segments view</a> in the web console.
The segments for the first 12 hours of 2015-09-12 are now gone:</p>
<p><img src="../assets/tutorial-retention-06.png" alt="New segments" title="New segments"></p>
<p>The resulting retention rule chain is the following:</p>
diff --git a/docs/24.0.0/tutorials/tutorial-rollup.html b/docs/24.0.0/tutorials/tutorial-rollup.html
index c68f3826..94e5c55a 100644
--- a/docs/24.0.0/tutorials/tutorial-rollup.html
+++ b/docs/24.0.0/tutorials/tutorial-rollup.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/24.0.0/tutorials/tutorial-sketches-theta.html b/docs/24.0.0/tutorials/tutorial-sketches-theta.html
index 50b0fd7e..e5b38b9d 100644
--- a/docs/24.0.0/tutorials/tutorial-sketches-theta.html
+++ b/docs/24.0.0/tutorials/tutorial-sketches-theta.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -139,7 +139,7 @@ It will also be helpful to have finished <a href="/docs/24.0.0/tutorials/tutoria
</code></pre>
<h2><a class="anchor" aria-hidden="true" id="ingest-data-using-theta-sketches"></a><a href="#ingest-data-using-theta-sketches" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13. [...]
<ol>
-<li>Navigate to the <strong>Load data</strong> wizard in the Druid console.</li>
+<li>Navigate to the <strong>Load data</strong> wizard in the web console.</li>
<li>Select <code>Paste data</code> as the data source and paste the given data:</li>
</ol>
<p><img src="../assets/tutorial-theta-01.png" alt="Load data view with pasted data"></p>
diff --git a/docs/24.0.0/tutorials/tutorial-transform-spec.html b/docs/24.0.0/tutorials/tutorial-transform-spec.html
index 184cb557..df1be99c 100644
--- a/docs/24.0.0/tutorials/tutorial-transform-spec.html
+++ b/docs/24.0.0/tutorials/tutorial-transform-spec.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -190,7 +190,7 @@ the <a href="/docs/24.0.0/tutorials/index.html">single-machine quickstart</a> an
Retrieved 3 rows <span class="hljs-keyword">in</span> 0.03s.
</code></pre>
<p>The "lion" row has been discarded, the <code>animal</code> column has been transformed, and we have both the original and transformed <code>number</code> column.</p>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-ingestion-spec.html"><span class="arrow-prev">← </span><span>Writing an ingestion spec</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/tutorial-kerberos-hadoop.html"><span>Kerberized HDFS deep storage</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#sample-data">Sample data</ [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/24.0.0/tutorials/tutorial-ingestion-spec.html"><span class="arrow-prev">← </span><span>Writing an ingestion spec</span></a><a class="docs-next button" href="/docs/24.0.0/tutorials/docker.html"><span>Tutorial: Run with Docker</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#sample-data">Sample data</a></li><li><a href="# [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/24.0.0/tutorials/tutorial-update-data.html b/docs/24.0.0/tutorials/tutorial-update-data.html
index 3a363cec..53fca8e7 100644
--- a/docs/24.0.0/tutorials/tutorial-update-data.html
+++ b/docs/24.0.0/tutorials/tutorial-update-data.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-elasticsearch.html b/docs/latest/comparisons/druid-vs-elasticsearch.html
index 5e935aa3..06c6bd1b 100644
--- a/docs/latest/comparisons/druid-vs-elasticsearch.html
+++ b/docs/latest/comparisons/druid-vs-elasticsearch.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-key-value.html b/docs/latest/comparisons/druid-vs-key-value.html
index c40811ab..526de6d9 100644
--- a/docs/latest/comparisons/druid-vs-key-value.html
+++ b/docs/latest/comparisons/druid-vs-key-value.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-kudu.html b/docs/latest/comparisons/druid-vs-kudu.html
index 2007b0b9..317a0bda 100644
--- a/docs/latest/comparisons/druid-vs-kudu.html
+++ b/docs/latest/comparisons/druid-vs-kudu.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-redshift.html b/docs/latest/comparisons/druid-vs-redshift.html
index 1849bde0..ee9d6093 100644
--- a/docs/latest/comparisons/druid-vs-redshift.html
+++ b/docs/latest/comparisons/druid-vs-redshift.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-spark.html b/docs/latest/comparisons/druid-vs-spark.html
index 5625a8c4..7c20eb83 100644
--- a/docs/latest/comparisons/druid-vs-spark.html
+++ b/docs/latest/comparisons/druid-vs-spark.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/comparisons/druid-vs-sql-on-hadoop.html b/docs/latest/comparisons/druid-vs-sql-on-hadoop.html
index d030ec47..1003670b 100644
--- a/docs/latest/comparisons/druid-vs-sql-on-hadoop.html
+++ b/docs/latest/comparisons/druid-vs-sql-on-hadoop.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/configuration/index.html b/docs/latest/configuration/index.html
index 0bc72f53..ad124502 100644
--- a/docs/latest/configuration/index.html
+++ b/docs/latest/configuration/index.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -1027,7 +1027,7 @@ Support for 64-bit floating point columns was released in Druid 0.11.0, so if yo
</table>
<h4><a class="anchor" aria-hidden="true" id="dynamic-configuration"></a><a href="#dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
<p>The Coordinator has dynamic configuration to change certain behavior on the fly.</p>
-<p>It is recommended that you use the <a href="/docs/latest/operations/druid-console.html">web console</a> to configure these parameters.
+<p>It is recommended that you use the <a href="/docs/latest/operations/web-console.html">web console</a> to configure these parameters.
However, if you need to do it via HTTP, the JSON object can be submitted to the Coordinator via a POST request at:</p>
<pre><code class="hljs">http:<span class="hljs-regexp">//</span><COORDINATOR_IP>:<PORT><span class="hljs-regexp">/druid/</span>coordinator<span class="hljs-regexp">/v1/</span>config
</code></pre>
@@ -1109,7 +1109,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
</tbody>
</table>
<h5><a class="anchor" aria-hidden="true" id="automatic-compaction-dynamic-configuration"></a><a href="#automatic-compaction-dynamic-configuration" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c [...]
-<p>You can set or update <a href="/docs/latest/ingestion/automatic-compaction.html">automatic compaction</a> properties dynamically using the
+<p>You can set or update <a href="/docs/latest/data-management/automatic-compaction.html">automatic compaction</a> properties dynamically using the
<a href="/docs/latest/operations/api-reference.html#automatic-compaction-configuration">Coordinator API</a> without restarting Coordinators.</p>
<p>For details about segment compaction, see <a href="/docs/latest/operations/segment-optimization.html">Segment size optimization</a>.</p>
<p>You can configure automatic compaction through the following properties:</p>
@@ -1121,7 +1121,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
<tr><td><code>dataSource</code></td><td>dataSource name to be compacted.</td><td>yes</td></tr>
<tr><td><code>taskPriority</code></td><td><a href="/docs/latest/ingestion/tasks.html#priority">Priority</a> of compaction task.</td><td>no (default = 25)</td></tr>
<tr><td><code>inputSegmentSizeBytes</code></td><td>Maximum number of total segment bytes processed per compaction task. Since a time chunk must be processed in its entirety, if the segments for a particular time chunk have a total size in bytes greater than this parameter, compaction will not run for that time chunk. Because each compaction task runs with a single thread, setting this value too far above 1–2GB will result in compaction tasks taking an excessive amount of time.</td><td>no [...]
-<tr><td><code>skipOffsetFromLatest</code></td><td>The offset for searching segments to be compacted in <a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a> duration format. Strongly recommended to set for realtime dataSources. See <a href="/docs/latest/ingestion/compaction.html#data-handling-with-compaction">Data handling with compaction</a>.</td><td>no (default = "P1D")</td></tr>
+<tr><td><code>skipOffsetFromLatest</code></td><td>The offset for searching segments to be compacted in <a href="https://en.wikipedia.org/wiki/ISO_8601">ISO 8601</a> duration format. Strongly recommended to set for realtime dataSources. See <a href="/docs/latest/data-management/compaction.html#data-handling-with-compaction">Data handling with compaction</a>.</td><td>no (default = "P1D")</td></tr>
<tr><td><code>tuningConfig</code></td><td>Tuning config for compaction tasks. See below <a href="#automatic-compaction-tuningconfig">Automatic compaction tuningConfig</a>.</td><td>no</td></tr>
<tr><td><code>taskContext</code></td><td><a href="/docs/latest/ingestion/tasks.html#context">Task context</a> for compaction tasks.</td><td>no</td></tr>
<tr><td><code>granularitySpec</code></td><td>Custom <code>granularitySpec</code>. See <a href="#automatic-compaction-granularityspec">Automatic compaction granularitySpec</a>.</td><td>No</td></tr>
@@ -1143,7 +1143,7 @@ However, if you need to do it via HTTP, the JSON object can be submitted to the
You may see this issue with streaming ingestion from Kafka and Kinesis, which ingest late-arriving data.</p>
<p>To mitigate this problem, set <code>skipOffsetFromLatest</code> to a value large enough so that arriving data tends to fall outside the offset value from the current time. This way you can avoid conflicts between compaction tasks and realtime ingestion tasks.
For example, if you want to skip over segments from thirty days prior to the end time of the most recent segment, assign <code>"skipOffsetFromLatest": "P30D"</code>.
-For more information, see <a href="/docs/latest/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
+For more information, see <a href="/docs/latest/data-management/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
<h6><a class="anchor" aria-hidden="true" id="automatic-compaction-tuningconfig"></a><a href="#automatic-compaction-tuningconfig" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S1 [...]
<p>Auto-compaction supports a subset of the <a href="/docs/latest/ingestion/native-batch.html#tuningconfig">tuningConfig for Parallel task</a>.
The below is a list of the supported configurations for auto-compaction.</p>
@@ -1161,7 +1161,7 @@ The below is a list of the supported configurations for auto-compaction.</p>
<tr><td><code>indexSpecForIntermediatePersists</code></td><td>Defines segment storage format options to be used at indexing time for intermediate persisted temporary segments. this can be used to disable dimension/metric compression on intermediate segments to reduce memory required for final merging. however, disabling compression on intermediate segments might increase page cache use while they are used before getting merged into final segment published, see <a href="/docs/latest/inges [...]
<tr><td><code>maxPendingPersists</code></td><td>Maximum number of persists that can be pending but not started. If this limit would be exceeded by a new intermediate persist, ingestion will block until the currently-running persist finishes. Maximum heap memory usage for indexing scales with <code>maxRowsInMemory</code> * (2 + <code>maxPendingPersists</code>).</td><td>no (default = 0, meaning one persist can be running concurrently with ingestion, and none can be queued up)</td></tr>
<tr><td><code>pushTimeout</code></td><td>Milliseconds to wait for pushing segments. It must be >= 0, where 0 means to wait forever.</td><td>no (default = 0)</td></tr>
-<tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="/docs/latest/ingestion/native-batch-simple-task.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>no (default is the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used)</td></tr>
+<tr><td><code>segmentWriteOutMediumFactory</code></td><td>Segment write-out medium to use when creating segments. See <a href="/docs/latest/ingestion/native-batch.html#segmentwriteoutmediumfactory">SegmentWriteOutMediumFactory</a>.</td><td>no (default is the value from <code>druid.peon.defaultSegmentWriteOutMediumFactory.type</code> is used)</td></tr>
<tr><td><code>maxNumConcurrentSubTasks</code></td><td>Maximum number of worker tasks which can be run in parallel at the same time. The supervisor task would spawn worker tasks up to <code>maxNumConcurrentSubTasks</code> regardless of the current available task slots. If this value is set to 1, the supervisor task processes data ingestion on its own instead of spawning worker tasks. If this value is set to too large, too many worker tasks can be created which might block other ingestion. [...]
<tr><td><code>maxRetry</code></td><td>Maximum number of retries on task failures.</td><td>no (default = 3)</td></tr>
<tr><td><code>maxNumSegmentsToMerge</code></td><td>Max limit for the number of segments that a single task can merge at the same time in the second phase. Used only with <code>hashed</code> or <code>single_dim</code> partitionsSpec.</td><td>no (default = 100)</td></tr>
@@ -1965,7 +1965,7 @@ line.</p>
<tr><td><code>druid.server.http.enableRequestLimit</code></td><td>If enabled, no requests would be queued in jetty queue and "HTTP 429 Too Many Requests" error response would be sent.</td><td>false</td></tr>
<tr><td><code>druid.server.http.defaultQueryTimeout</code></td><td>Query timeout in millis, beyond which unfinished queries will be cancelled</td><td>300000</td></tr>
<tr><td><code>druid.server.http.maxScatterGatherBytes</code></td><td>Maximum number of bytes gathered from data processes such as Historicals and realtime processes to execute a query. Queries that exceed this limit will fail. This is an advance configuration that allows to protect in case Broker is under heavy load and not utilizing the data gathered in memory fast enough and leading to OOMs. This limit can be further reduced at query time using <code>maxScatterGatherBytes</code> in the [...]
-<tr><td><code>druid.server.http.maxSubqueryRows</code></td><td>Maximum number of rows from all subqueries per query. Druid stores the subquery rows in temporary tables that live in the Java heap. <code>druid.server.http.maxSubqueryRows</code> is a guardrail to prevent the system from exhausting available heap. When a subquery exceeds the row limit, Druid throws a resource limit exceeded exception: "Subquery generated results beyond maximum."<br><br>It is a good practice to avoi [...]
+<tr><td><code>druid.server.http.maxSubqueryRows</code></td><td>Maximum number of rows from all subqueries per query. Druid stores the subquery rows in temporary tables that live in the Java heap. <code>druid.server.http.maxSubqueryRows</code> is a guardrail to prevent the system from exhausting available heap. When a subquery exceeds the row limit, Druid throws a resource limit exceeded exception: "Subquery generated results beyond maximum."<br /><br />It is a good practice to [...]
<tr><td><code>druid.server.http.gracefulShutdownTimeout</code></td><td>The maximum amount of time Jetty waits after receiving shutdown signal. After this timeout the threads will be forcefully shutdown. This allows any queries that are executing to complete(Only values greater than zero are valid).</td><td><code>PT30S</code></td></tr>
<tr><td><code>druid.server.http.unannouncePropagationDelay</code></td><td>How long to wait for ZooKeeper unannouncements to propagate before shutting down Jetty. This is a minimum and <code>druid.server.http.gracefulShutdownTimeout</code> does not start counting down until after this period elapses.</td><td><code>PT0S</code> (do not wait)</td></tr>
<tr><td><code>druid.server.http.maxQueryTimeout</code></td><td>Maximum allowed value (in milliseconds) for <code>timeout</code> parameter. See <a href="/docs/latest/querying/query-context.html">query-context</a> to know more about <code>timeout</code>. Query is rejected if the query context <code>timeout</code> is greater than this value.</td><td>Long.MAX_VALUE</td></tr>
@@ -2070,7 +2070,7 @@ queries in order to avoid running as a default priority of 0.</p>
<tr><td><code>druid.sql.planner.authorizeSystemTablesDirectly</code></td><td>If true, Druid authorizes queries against any of the system schema tables (<code>sys</code> in SQL) as <code>SYSTEM_TABLE</code> resources which require <code>READ</code> access, in addition to permissions based content filtering.</td><td>false</td></tr>
<tr><td><code>druid.sql.planner.useNativeQueryExplain</code></td><td>If true, <code>EXPLAIN PLAN FOR</code> will return the explain plan as a JSON representation of equivalent native query(s), else it will return the original version of explain plan generated by Calcite. It can be overridden per query with <code>useNativeQueryExplain</code> context key.</td><td>true</td></tr>
<tr><td><code>druid.sql.planner.maxNumericInFilters</code></td><td>Max limit for the amount of numeric values that can be compared for a string type dimension when the entire SQL WHERE clause of a query translates to an <a href="/docs/latest/querying/filters.html#or">OR</a> of <a href="/docs/latest/querying/filters.html#bound-filter">Bound filter</a>. By default, Druid does not restrict the amount of numeric Bound Filters on String columns, although this situation may block other queries [...]
-<tr><td><code>druid.sql.approxCountDistinct.function</code></td><td>Implementation to use for the <a href="/docs/latest/querying/sql-aggregations.html"><code>APPROX_COUNT_DISTINCT</code> function</a>. Without extensions loaded, the only valid value is <code>APPROX_COUNT_DISTINCT_BUILTIN</code> (a HyperLogLog, or HLL, based implementation). If the <a href="/docs/latest/development/extensions-core/datasketches-extension.html">DataSketches extension</a> is loaded, this can also be <code>APP [...]
+<tr><td><code>druid.sql.approxCountDistinct.function</code></td><td>Implementation to use for the <a href="/docs/latest/querying/sql-aggregations.html"><code>APPROX_COUNT_DISTINCT</code> function</a>. Without extensions loaded, the only valid value is <code>APPROX_COUNT_DISTINCT_BUILTIN</code> (a HyperLogLog, or HLL, based implementation). If the <a href="/docs/latest/development/extensions-core/datasketches-extension.html">DataSketches extension</a> is loaded, this can also be <code>APP [...]
</tbody>
</table>
<blockquote>
diff --git a/docs/latest/configuration/logging.html b/docs/latest/configuration/logging.html
index 8b7d42fd..ed720a52 100644
--- a/docs/latest/configuration/logging.html
+++ b/docs/latest/configuration/logging.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -176,7 +176,7 @@ about <a href="/docs/latest/operations/java.html#strong-encapsulation">Java stro
<span class="hljs-tag"></<span class="hljs-name">Loggers</span>></span>
<span class="hljs-tag"></<span class="hljs-name">Configuration</span>></span>
</span></code></pre>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/development/extensions.html"><span class="arrow-prev">← </span><span>Extensions</span></a><a class="docs-next button" href="/docs/latest/operations/druid-console.html"><span>Druid console</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#log-directory">Log directory</a></li><li><a href="#all-in-one-start-commands" [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/development/extensions.html"><span class="arrow-prev">← </span><span>Extensions</span></a><a class="docs-next button" href="/docs/latest/operations/web-console.html"><span>Web console</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#log-directory">Log directory</a></li><li><a href="#all-in-one-start-commands">All [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/latest/data-management/automatic-compaction.html b/docs/latest/data-management/automatic-compaction.html
new file mode 100644
index 00000000..e3c1ee0d
--- /dev/null
+++ b/docs/latest/data-management/automatic-compaction.html
@@ -0,0 +1,232 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Automatic compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/automatic-compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" c [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/automatic-compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Automatic compaction</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<p>In Apache Druid, compaction is a special type of ingestion task that reads data from a Druid datasource and writes it back into the same datasource. A common use case for this is to <a href="/docs/latest/operations/segment-optimization.html">optimally size segments</a> after ingestion to improve query performance. Automatic compaction, or auto-compaction, refers to the system for automatic execution of compaction tasks managed by the <a href="/docs/latest/design/coordinator.html">Drui [...]
+<p>The Coordinator <a href="/docs/latest/configuration/index.html#coordinator-operation">indexing period</a>, <code>druid.coordinator.period.indexingPeriod</code>, controls the frequency of compaction tasks.
+The default indexing period is 30 minutes, meaning that the Coordinator first checks for segments to compact at most 30 minutes from when auto-compaction is enabled.
+This time period affects other Coordinator duties including merge and conversion tasks.
+To configure the auto-compaction time period without interfering with <code>indexingPeriod</code>, see <a href="#set-frequency-of-compaction-runs">Set frequency of compaction runs</a>.</p>
+<p>At every invocation of auto-compaction, the Coordinator initiates a <a href="/docs/latest/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search</a> to determine eligible segments to compact.
+When there are eligible segments to compact, the Coordinator issues compaction tasks based on available worker capacity.
+If a compaction task takes longer than the indexing period, the Coordinator waits for it to finish before resuming the period for segment search.</p>
+<p>As a best practice, you should set up auto-compaction for all Druid datasources. You can run compaction tasks manually for cases where you want to allocate more system resources. For example, you may choose to run multiple compaction tasks in parallel to compact an existing datasource for the first time. See <a href="/docs/latest/data-management/compaction.html">Compaction</a> for additional details and use cases.</p>
+<p>This topic guides you through setting up automatic compaction for your Druid cluster. See the <a href="#examples">examples</a> for common use cases for automatic compaction.</p>
+<h2><a class="anchor" aria-hidden="true" id="enable-automatic-compaction"></a><a href="#enable-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
+<p>You can enable automatic compaction for a datasource using the web console or programmatically via an API.
+This process differs for manual compaction tasks, which can be submitted from the <a href="/docs/latest/operations/web-console.html">Tasks view of the web console</a> or the <a href="/docs/latest/operations/api-reference.html#post-5">Tasks API</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="web-console"></a><a href="#web-console" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42 [...]
+<p>Use the web console to enable automatic compaction for a datasource as follows.</p>
+<ol>
+<li>Click <strong>Datasources</strong> in the top-level navigation.</li>
+<li>In the <strong>Compaction</strong> column, click the edit icon for the datasource to compact.</li>
+<li>In the <strong>Compaction config</strong> dialog, configure the auto-compaction settings. The dialog offers a form view as well as a JSON view. Editing the form updates the JSON specification, and editing the JSON updates the form field, if present. Form fields not present in the JSON indicate default values. You may add additional properties to the JSON for auto-compaction settings not displayed in the form. See <a href="#configure-automatic-compaction">Configure automatic compactio [...]
+<li>Click <strong>Submit</strong>.</li>
+<li>Refresh the <strong>Datasources</strong> view. The <strong>Compaction</strong> column for the datasource changes from “Not enabled” to “Awaiting first run.”</li>
+</ol>
+<p>The following screenshot shows the compaction config dialog for a datasource with auto-compaction enabled.
+<img src="../assets/compaction-config.png" alt="Compaction config in web console"></p>
+<p>To disable auto-compaction for a datasource, click <strong>Delete</strong> from the <strong>Compaction config</strong> dialog. Druid does not retain your auto-compaction configuration.</p>
+<h3><a class="anchor" aria-hidden="true" id="compaction-configuration-api"></a><a href="#compaction-configuration-api" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
+<p>Use the <a href="/docs/latest/operations/api-reference.html#automatic-compaction-status">Coordinator API</a> to configure automatic compaction.
+To enable auto-compaction for a datasource, create a JSON object with the desired auto-compaction settings.
+See <a href="#configure-automatic-compaction">Configure automatic compaction</a> for the syntax of an auto-compaction spec.
+Send the JSON object as a payload in a <a href="/docs/latest/operations/api-reference.html#post-4"><code>POST</code> request</a> to <code>/druid/coordinator/v1/config/compaction</code>.
+The following example configures auto-compaction for the <code>wikipedia</code> datasource:</p>
+<pre><code class="hljs css language-sh">curl --location --request POST <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction'</span> \
+--header <span class="hljs-string">'Content-Type: application/json'</span> \
+--data-raw <span class="hljs-string">'{
+ "dataSource": "wikipedia",
+ "granularitySpec": {
+ "segmentGranularity": "DAY"
+ }
+}'</span>
+</code></pre>
+<p>To disable auto-compaction for a datasource, send a <a href="/docs/latest/operations/api-reference.html#delete-1"><code>DELETE</code> request</a> to <code>/druid/coordinator/v1/config/compaction/{dataSource}</code>. Replace <code>{dataSource}</code> with the name of the datasource for which to disable auto-compaction. For example:</p>
+<pre><code class="hljs css language-sh">curl --location --request DELETE <span class="hljs-string">'http://localhost:8081/druid/coordinator/v1/config/compaction/wikipedia'</span>
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="configure-automatic-compaction"></a><a href="#configure-automatic-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>You can configure automatic compaction dynamically without restarting Druid.
+The automatic compaction system uses the following syntax:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
+ <span class="hljs-attr">"ioConfig"</span>: <IO config>,
+ <span class="hljs-attr">"dimensionsSpec"</span>: <custom dimensionsSpec>,
+ <span class="hljs-attr">"transformSpec"</span>: <custom transformSpec>,
+ <span class="hljs-attr">"metricsSpec"</span>: <custom metricsSpec>,
+ <span class="hljs-attr">"tuningConfig"</span>: <parallel indexing task tuningConfig>,
+ <span class="hljs-attr">"granularitySpec"</span>: <compaction task granularitySpec>,
+ <span class="hljs-attr">"skipOffsetFromLatest"</span>: <time period to avoid compaction>,
+ <span class="hljs-attr">"taskPriority"</span>: <compaction task priority>,
+ <span class="hljs-attr">"taskContext"</span>: <task context>
+}
+</code></pre>
+<p>Most fields in the auto-compaction configuration correlate to a typical <a href="/docs/latest/ingestion/ingestion-spec.html">Druid ingestion spec</a>.
+The following properties only apply to auto-compaction:</p>
+<ul>
+<li><code>skipOffsetFromLatest</code></li>
+<li><code>taskPriority</code></li>
+<li><code>taskContext</code></li>
+</ul>
+<p>Since the automatic compaction system provides a management layer on top of manual compaction tasks,
+the auto-compaction configuration does not include task-specific properties found in a typical Druid ingestion spec.
+The following properties are automatically set by the Coordinator:</p>
+<ul>
+<li><code>type</code>: Set to <code>compact</code>.</li>
+<li><code>id</code>: Generated using the task type, datasource name, interval, and timestamp. The task ID is prefixed with <code>coordinator-issued</code>.</li>
+<li><code>context</code>: Set according to the user-provided <code>taskContext</code>.</li>
+</ul>
+<p>For more details on each of the specs in an auto-compaction configuration, see <a href="/docs/latest/configuration/index.html#automatic-compaction-dynamic-configuration">Automatic compaction dynamic configuration</a>.</p>
+<h3><a class="anchor" aria-hidden="true" id="avoid-conflicts-with-ingestion"></a><a href="#avoid-conflicts-with-ingestion" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 1 [...]
+<p>Compaction tasks may be interrupted when they interfere with ingestion. For example, this occurs when an ingestion task needs to write data to a segment for a time interval locked for compaction. If there are continuous failures that prevent compaction from making progress, consider one of the following strategies:</p>
+<ul>
+<li>Set <code>skipOffsetFromLatest</code> to reduce the chance of conflicts between ingestion and compaction. See more details in this section below.</li>
+<li>Increase the priority value of compaction tasks relative to ingestion tasks. Only recommended for advanced users. This approach can cause ingestion jobs to fail or lag. To change the priority of compaction tasks, set <code>taskPriority</code> to the desired priority value in the auto-compaction configuration. For details on the priority values of different task types, see <a href="/docs/latest/ingestion/tasks.html#lock-priority">Lock priority</a>.</li>
+</ul>
+<p>The Coordinator compacts segments from newest to oldest. In the auto-compaction configuration, you can set a time period, relative to the end time of the most recent segment, for segments that should not be compacted. Assign this value to <code>skipOffsetFromLatest</code>. Note that this offset is not relative to the current time but to the latest segment time. For example, if you want to skip over segments from five days prior to the end time of the most recent segment, assign <code> [...]
+<p>To set <code>skipOffsetFromLatest</code>, consider how frequently you expect the stream to receive late arriving data. If your stream only occasionally receives late arriving data, the auto-compaction system robustly compacts your data even though data is ingested outside the <code>skipOffsetFromLatest</code> window. For most realtime streaming ingestion use cases, it is reasonable to set <code>skipOffsetFromLatest</code> to a few hours or a day.</p>
+<h3><a class="anchor" aria-hidden="true" id="set-frequency-of-compaction-runs"></a><a href="#set-frequency-of-compaction-runs" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13. [...]
+<p>If you want the Coordinator to check for compaction more frequently than its indexing period, create a separate group to handle compaction duties.
+Set the time period of the duty group in the <code>coordinator/runtime.properties</code> file.
+The following example shows how to create a duty group named <code>compaction</code> and set the auto-compaction period to 1 minute:</p>
+<pre><code class="hljs"><span class="hljs-attr">druid.coordinator.dutyGroups</span>=[<span class="hljs-string">"compaction"</span>]
+<span class="hljs-attr">druid.coordinator.compaction.duties</span>=[<span class="hljs-string">"compactSegments"</span>]
+<span class="hljs-attr">druid.coordinator.compaction.period</span>=PT60S
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="view-automatic-compaction-statistics"></a><a href="#view-automatic-compaction-statistics" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 [...]
+<p>After the Coordinator has initiated auto-compaction, you can view compaction statistics for the datasource, including the number of bytes, segments, and intervals already compacted and those awaiting compaction. The Coordinator also reports the total bytes, segments, and intervals not eligible for compaction in accordance with its <a href="/docs/latest/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>.</p>
+<p>In the web console, the Datasources view displays auto-compaction statistics. The Tasks view shows the task information for compaction tasks that were triggered by the automatic compaction system.</p>
+<p>To get statistics by API, send a <a href="/docs/latest/operations/api-reference.html#get-10"><code>GET</code> request</a> to <code>/druid/coordinator/v1/compaction/status</code>. To filter the results to a particular datasource, pass the datasource name as a query parameter to the request—for example, <code>/druid/coordinator/v1/compaction/status?dataSource=wikipedia</code>.</p>
+<h2><a class="anchor" aria-hidden="true" id="examples"></a><a href="#examples" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 [...]
+<p>The following examples demonstrate potential use cases in which auto-compaction may improve your Druid performance. See more details in <a href="/docs/latest/data-management/compaction.html#compaction-strategies">Compaction strategies</a>. The examples in this section do not change the underlying data.</p>
+<h3><a class="anchor" aria-hidden="true" id="change-segment-granularity"></a><a href="#change-segment-granularity" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
+<p>You have a stream set up to ingest data with <code>HOUR</code> segment granularity into the <code>wikistream</code> datasource. You notice that your Druid segments are smaller than the <a href="/docs/latest/operations/segment-optimization.html">recommended segment size</a> of 5 million rows per segment. You wish to automatically compact segments to <code>DAY</code> granularity while leaving the latest week of data <em>not</em> compacted because your stream consistently receives data w [...]
+<p>The following auto-compaction configuration compacts existing <code>HOUR</code> segments into <code>DAY</code> segments while leaving the latest week of data not compacted:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikistream"</span>,
+ <span class="hljs-attr">"granularitySpec"</span>: {
+ <span class="hljs-attr">"segmentGranularity"</span>: <span class="hljs-string">"DAY"</span>
+ },
+ <span class="hljs-attr">"skipOffsetFromLatest"</span>: <span class="hljs-string">"P1W"</span>,
+}
+</code></pre>
+<h3><a class="anchor" aria-hidden="true" id="update-partitioning-scheme"></a><a href="#update-partitioning-scheme" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H [...]
+<p>For your <code>wikipedia</code> datasource, you want to optimize segment access when regularly ingesting data without compromising compute time when querying the data. Your ingestion spec for batch append uses <a href="/docs/latest/ingestion/native-batch.html#dynamic-partitioning">dynamic partitioning</a> to optimize for write-time operations, while your stream ingestion partitioning is configured by the stream service. You want to implement auto-compaction to reorganize the data with [...]
+<p>The following auto-compaction configuration compacts updates the <code>wikipedia</code> segments to use multi-dimension range partitioning:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"dataSource"</span>: <span class="hljs-string">"wikipedia"</span>,
+ <span class="hljs-attr">"tuningConfig"</span>: {
+ <span class="hljs-attr">"partitionsSpec"</span>: {
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"range"</span>,
+ <span class="hljs-attr">"partitionDimensions"</span>: [
+ <span class="hljs-string">"channel"</span>,
+ <span class="hljs-string">"countryName"</span>,
+ <span class="hljs-string">"namespace"</span>
+ ],
+ <span class="hljs-attr">"targetRowsPerSegment"</span>: <span class="hljs-number">5000000</span>
+ }
+ }
+}
+</code></pre>
+<h2><a class="anchor" aria-hidden="true" id="learn-more"></a><a href="#learn-more" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1 [...]
+<p>See the following topics for more information:</p>
+<ul>
+<li><a href="/docs/latest/data-management/compaction.html">Compaction</a> for an overview of compaction and how to set up manual compaction in Druid.</li>
+<li><a href="/docs/latest/operations/segment-optimization.html">Segment optimization</a> for guidance on evaluating and optimizing Druid segment size.</li>
+<li><a href="/docs/latest/design/coordinator.html#automatic-compaction">Coordinator process</a> for details on how the Coordinator plans compaction tasks.</li>
+</ul>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/data-management/compaction.html"><span class="arrow-prev">← </span><span>Compaction</span></a><a class="docs-next button" href="/docs/latest/querying/sql.html"><span>Overview and syntax</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#enable-automatic-compaction">Enable automatic compaction</a><ul class="toc-head [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/latest/ingestion/compaction.html b/docs/latest/data-management/compaction.html
similarity index 54%
copy from docs/latest/ingestion/compaction.html
copy to docs/latest/data-management/compaction.html
index a68a11a6..20a09bc9 100644
--- a/docs/latest/ingestion/compaction.html
+++ b/docs/latest/data-management/compaction.html
@@ -1,9 +1,9 @@
-<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/ingestion/compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Use case [...]
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Compaction · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/compaction.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="Defines compaction and automatic compaction (auto-compaction or autocompaction) for segment optimization. Us [...]
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -59,7 +59,7 @@
};
}
});
- </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/ingestion/compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Compaction</h1></header><article><div><span><!--
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/compaction.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Compaction</h1></header><article><div><span><!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
@@ -97,7 +97,7 @@ Query performance in Apache Druid depends on optimally sized segments. Compactio
<p>Compaction does not improve performance in all situations. For example, if you rewrite your data with each ingestion task, you don't need to use compaction. See <a href="/docs/latest/operations/segment-optimization.html">Segment optimization</a> for additional guidance to determine if compaction will help in your environment.</p>
<h2><a class="anchor" aria-hidden="true" id="types-of-compaction"></a><a href="#types-of-compaction" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.2 [...]
<p>You can configure the Druid Coordinator to perform automatic compaction, also called auto-compaction, for a datasource. Using its <a href="/docs/latest/design/coordinator.html#segment-search-policy-in-automatic-compaction">segment search policy</a>, the Coordinator periodically identifies segments for compaction starting from newest to oldest. When the Coordinator discovers segments that have not been compacted or segments that were compacted with a different or changed spec, it submi [...]
-<p>Automatic compaction works in most use cases and should be your first option. To learn more, see <a href="/docs/latest/ingestion/automatic-compaction.html">Automatic compaction</a>.</p>
+<p>Automatic compaction works in most use cases and should be your first option. To learn more, see <a href="/docs/latest/data-management/automatic-compaction.html">Automatic compaction</a>.</p>
<p>In cases where you require more control over compaction, you can manually submit compaction tasks. For example:</p>
<ul>
<li>Automatic compaction is running into the limit of task slots available to it, so tasks are waiting for previous automatic compaction tasks to complete. Manual compaction can use all available task slots, therefore you can complete compaction more quickly by submitting more concurrent tasks for more intervals.</li>
@@ -112,7 +112,7 @@ Query performance in Apache Druid depends on optimally sized segments. Compactio
</blockquote>
<p>If an ingestion task needs to write data to a segment for a time interval locked for compaction, by default the ingestion task supersedes the compaction task and the compaction task fails without finishing. For manual compaction tasks, you can adjust the input spec interval to avoid conflicts between ingestion and compaction. For automatic compaction, you can set the <code>skipOffsetFromLatest</code> key to adjust the auto-compaction starting point from the current time to reduce the [...]
Another option is to set the compaction task to higher priority than the ingestion task.
-For more information, see <a href="/docs/latest/ingestion/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
+For more information, see <a href="/docs/latest/data-management/automatic-compaction.html#avoid-conflicts-with-ingestion">Avoid conflicts with ingestion</a>.</p>
<h3><a class="anchor" aria-hidden="true" id="segment-granularity-handling"></a><a href="#segment-granularity-handling" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 [...]
<p>Unless you modify the segment granularity in <a href="#compaction-granularity-spec"><code>granularitySpec</code></a>, Druid attempts to retain the granularity for the compacted segments. When segments have different segment granularities with no overlap in interval Druid creates a separate compaction task for each to retain the segment granularity in the compacted segment.</p>
<p>If segments have different segment granularities before compaction but there is some overlap in interval, Druid attempts find start and end of the overlapping interval and uses the closest segment granularity level for the compacted segment.</p>
@@ -171,10 +171,10 @@ You can check that your segments are rolled up or not by using <a href="/docs/la
<blockquote>
<p>You can run multiple compaction tasks in parallel. For example, if you want to compact the data for a year, you are not limited to running a single task for the entire year. You can run 12 compaction tasks with month-long intervals.</p>
</blockquote>
-<p>A compaction task internally generates an <code>index</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/latest/ingestion/native-batch-input-sources.html">DruidInputSource</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p>
+<p>A compaction task internally generates an <code>index</code> task spec for performing compaction work with some fixed parameters. For example, its <code>inputSource</code> is always the <a href="/docs/latest/ingestion/native-batch-input-sources.html"><code>druid</code> input source</a>, and <code>dimensionsSpec</code> and <code>metricsSpec</code> include all dimensions and metrics of the input segments by default.</p>
<p>Compaction tasks exit without doing anything and issue a failure status code in either of the following cases:</p>
<ul>
-<li>If the interval you specify has no data segments loaded<br></li>
+<li>If the interval you specify has no data segments loaded<br /></li>
<li>If the interval you specify is empty.</li>
</ul>
<p>Note that the metadata between input segments and the resulting compacted segments may differ if the metadata among the input segments differs as well. If all input segments have the same metadata, however, the resulting output segment will have the same metadata as all input segments.</p>
@@ -207,7 +207,7 @@ If you don't specify <code>granularitySpec</code>, Druid retains the original se
<tbody>
<tr><td><code>type</code></td><td>Task type. Set the value to <code>compact</code>.</td><td>none</td><td>Yes</td></tr>
<tr><td><code>inputSpec</code></td><td>Specification of the target <a href="#interval-inputspec">intervals</a> or <a href="#segments-inputspec">segments</a>.</td><td>none</td><td>Yes</td></tr>
-<tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br>- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br>- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br>If compaction fails, Druid does not change any of the existing segments.<br><strong>WARNING</strong>: <code>dropExisting</code> in <code>io [...]
+<tr><td><code>dropExisting</code></td><td>If <code>true</code>, the task replaces all existing segments fully contained by either of the following:<br />- the <code>interval</code> in the <code>interval</code> type <code>inputSpec</code>.<br />- the umbrella interval of the <code>segments</code> in the <code>segment</code> type <code>inputSpec</code>.<br />If compaction fails, Druid does not change any of the existing segments.<br /><strong>WARNING</strong>: <code>dropExisting</code> in [...]
</tbody>
</table>
<p>Druid supports two supported <code>inputSpec</code> formats:</p>
@@ -265,9 +265,9 @@ If you don't specify <code>granularitySpec</code>, Druid retains the original se
<p>See the following topics for more information:</p>
<ul>
<li><a href="/docs/latest/operations/segment-optimization.html">Segment optimization</a> for guidance to determine if compaction will help in your case.</li>
-<li><a href="/docs/latest/ingestion/automatic-compaction.html">Automatic compaction</a> for how to enable and configure automatic compaction.</li>
+<li><a href="/docs/latest/data-management/automatic-compaction.html">Automatic compaction</a> for how to enable and configure automatic compaction.</li>
</ul>
-</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/ingestion/data-management.html"><span class="arrow-prev">← </span><span>Data management</span></a><a class="docs-next button" href="/docs/latest/ingestion/automatic-compaction.html"><span>Automatic compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#compaction-strategies">Compaction strategies</a></li><li [...]
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/data-management/schema-changes.html"><span class="arrow-prev">← </span><span>Schema changes</span></a><a class="docs-next button" href="/docs/latest/data-management/automatic-compaction.html"><span>Automatic compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#compaction-strategies">Compaction strategies</ [...]
document.addEventListener('keyup', function(e) {
if (e.target !== document.body) {
return;
diff --git a/docs/latest/data-management/delete.html b/docs/latest/data-management/delete.html
new file mode 100644
index 00000000..7682fbb3
--- /dev/null
+++ b/docs/latest/data-management/delete.html
@@ -0,0 +1,156 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data deletion · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/delete.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><me [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/delete.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data deletion</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="by-time-range-manually"></a><a href="#by-time-range-manually" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0 [...]
+<p>Apache Druid stores data <a href="/docs/latest/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> and supports
+deleting data for time chunks by dropping segments. This is a fast, metadata-only operation.</p>
+<p>Deletion by time range happens in two steps:</p>
+<ol>
+<li>Segments to be deleted must first be marked as <a href="/docs/latest/design/architecture.html#segment-lifecycle">"unused"</a>. This can
+happen when a segment is dropped by a <a href="/docs/latest/operations/rule-configuration.html">drop rule</a> or when you manually mark a
+segment unused through the Coordinator API or web console. This is a soft delete: the data is not available for
+querying, but the segment files remains in deep storage, and the segment records remains in the metadata store.</li>
+<li>Once a segment is marked "unused", you can use a <a href="#kill-task"><code>kill</code> task</a> to permanently delete the segment file from
+deep storage and remove its record from the metadata store. This is a hard delete: the data is unrecoverable unless
+you have a backup.</li>
+</ol>
+<p>For documentation on disabling segments using the Coordinator API, see the
+<a href="/docs/latest/operations/api-reference.html#coordinator-datasources">Coordinator API reference</a>.</p>
+<p>A data deletion tutorial is available at <a href="/docs/latest/tutorials/tutorial-delete-data.html">Tutorial: Deleting data</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="by-time-range-automatically"></a><a href="#by-time-range-automatically" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 1 [...]
+<p>Druid supports <a href="/docs/latest/operations/rule-configuration.html">load and drop rules</a>, which are used to define intervals of time
+where data should be preserved, and intervals where data should be discarded. Data that falls under a drop rule is
+marked unused, in the same manner as if you <a href="#by-time-range-manually">manually mark that time range unused</a>. This is a
+fast, metadata-only operation.</p>
+<p>Data that is dropped in this way is marked unused, but remains in deep storage. To permanently delete it, use a
+<a href="#kill-task"><code>kill</code> task</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="specific-records"></a><a href="#specific-records" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2. [...]
+<p>Druid supports deleting specific records using <a href="/docs/latest/data-management/update.html#reindex">reindexing</a> with a filter. The filter specifies which
+data remains after reindexing, so it must be the inverse of the data you want to delete. Because segments must be
+rewritten to delete data in this way, it can be a time-consuming operation.</p>
+<p>For example, to delete records where <code>userName</code> is <code>'bob'</code> with native batch indexing, use a
+<a href="/docs/latest/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> with filter <code>{"type": "not", "field": {"type": "selector", "dimension": "userName", "value": "bob"}}</code>.</p>
+<p>To delete the same records using SQL, use <a href="/docs/latest/multi-stage-query/concepts.html#replace">REPLACE</a> with <code>WHERE userName <> 'bob'</code>.</p>
+<p>To reindex using <a href="/docs/latest/ingestion/native-batch.html">native batch</a>, use the <a href="/docs/latest/ingestion/native-batch-input-sources.html#druid-input-source"><code>druid</code> input
+source</a>. If needed,
+<a href="/docs/latest/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> can be used to filter or modify data during the
+reindexing job. To reindex with SQL, use <a href="/docs/latest/multi-stage-query/reference.html#replace"><code>REPLACE <table> OVERWRITE</code></a>
+with <code>SELECT ... FROM <table></code>. (Druid does not have <code>UPDATE</code> or <code>ALTER TABLE</code> statements.) Any SQL SELECT query can be
+used to filter, modify, or enrich the data during the reindexing job.</p>
+<p>Data that is deleted in this way is marked unused, but remains in deep storage. To permanently delete it, use a <a href="#kill-task"><code>kill</code>
+task</a>.</p>
+<h2><a class="anchor" aria-hidden="true" id="entire-table"></a><a href="#entire-table" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>Deleting an entire table works the same way as <a href="#by-time-range-manually">deleting part of a table by time range</a>. First,
+mark all segments unused using the Coordinator API or web console. Then, optionally, delete it permanently using a
+<a href="#kill-task"><code>kill</code> task</a>.</p>
+<p><a name="kill-task"></a></p>
+<h2><a class="anchor" aria-hidden="true" id="permanently-kill-task"></a><a href="#permanently-kill-task" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>Data that has been overwritten or soft-deleted still remains as segments that have been marked unused. You can use a
+<code>kill</code> task to permanently delete this data.</p>
+<p>The available grammar is:</p>
+<pre><code class="hljs css language-json">{
+ <span class="hljs-attr">"type"</span>: <span class="hljs-string">"kill"</span>,
+ <span class="hljs-attr">"id"</span>: <task_id>,
+ <span class="hljs-attr">"dataSource"</span>: <task_datasource>,
+ <span class="hljs-attr">"interval"</span> : <all_unused_segments_in_this_interval_will_die!>,
+ <span class="hljs-attr">"context"</span>: <task context>
+}
+</code></pre>
+<p><strong>WARNING:</strong> The <code>kill</code> task permanently removes all information about the affected segments from the metadata store and
+deep storage. This operation cannot be undone.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/data-management/update.html"><span class="arrow-prev">← </span><span>Data updates</span></a><a class="docs-next button" href="/docs/latest/data-management/schema-changes.html"><span>Schema changes</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#by-time-range-manually">By time range, manually</a></li><li><a href= [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/latest/data-management/index.html b/docs/latest/data-management/index.html
new file mode 100644
index 00000000..dbf96869
--- /dev/null
+++ b/docs/latest/data-management/index.html
@@ -0,0 +1,109 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data management · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/index.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><m [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/index.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data management</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<p>Apache Druid stores data <a href="/docs/latest/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> in immutable
+files called <a href="/docs/latest/design/segments.html">segments</a>. Data management operations involving replacing, or deleting,
+these segments include:</p>
+<ul>
+<li><a href="/docs/latest/data-management/update.html">Updates</a> to existing data.</li>
+<li><a href="/docs/latest/data-management/delete.html">Deletion</a> of existing data.</li>
+<li><a href="/docs/latest/data-management/schema-changes.html">Schema changes</a> for new and existing data.</li>
+<li><a href="/docs/latest/data-management/compaction.html">Compaction</a> and <a href="/docs/latest/data-management/automatic-compaction.html">automatic compaction</a>, which reindex existing data to
+optimize storage footprint and performance.</li>
+</ul>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/ingestion/faq.html"><span class="arrow-prev">← </span><span>Troubleshooting FAQ</span></a><a class="docs-next button" href="/docs/latest/data-management/update.html"><span>Data updates</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"></nav></div><footer class="nav-footer druid-footer" id="footer"><div class="container"><div class="text-center"><p><a h [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/latest/data-management/schema-changes.html b/docs/latest/data-management/schema-changes.html
new file mode 100644
index 00000000..cb45615d
--- /dev/null
+++ b/docs/latest/data-management/schema-changes.html
@@ -0,0 +1,110 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Schema changes · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/schema-changes.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0 [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/schema-changes.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Schema changes</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="for-new-data"></a><a href="#for-new-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83. [...]
+<p>Apache Druid allows you to provide a new schema for new data without the need to update the schema of any existing data.
+It is sufficient to update your supervisor spec, if using <a href="/docs/latest/ingestion/index.html#streaming">streaming ingestion</a>, or to
+provide the new schema the next time you do a <a href="/docs/latest/ingestion/index.html#batch">batch ingestion</a>. This is made possible by
+the fact that each <a href="/docs/latest/design/architecture.html#datasources-and-segments">segment</a>, at the time it is created, stores a
+copy of its own schema. Druid reconciles all of these individual segment schemas automatically at query time.</p>
+<h2><a class="anchor" aria-hidden="true" id="for-existing-data"></a><a href="#for-existing-data" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2- [...]
+<p>Schema changes are sometimes necessary for existing data. For example, you may want to change the type of a column in
+previously-ingested data, or drop a column entirely. Druid handles this using <a href="/docs/latest/data-management/update.html">reindexing</a>, the same method
+it uses to handle updates of existing data. Reindexing involves rewriting all affected segments and can be a
+time-consuming operation.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/data-management/delete.html"><span class="arrow-prev">← </span><span>Data deletion</span></a><a class="docs-next button" href="/docs/latest/data-management/compaction.html"><span>Compaction</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#for-new-data">For new data</a></li><li><a href="#for-existing-data">For exi [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/latest/data-management/update.html b/docs/latest/data-management/update.html
new file mode 100644
index 00000000..b39c88ab
--- /dev/null
+++ b/docs/latest/data-management/update.html
@@ -0,0 +1,138 @@
+<!DOCTYPE html><html lang="en"><head><meta charSet="utf-8"/><meta http-equiv="X-UA-Compatible" content="IE=edge"/><title>Data updates · Apache Druid</title><meta name="viewport" content="width=device-width"/><link rel="canonical" href="https://druid.apache.org/docs/latest/data-management/update.html"/><meta name="generator" content="Docusaurus"/><meta name="description" content="<!--"/><meta name="docsearch:language" content="en"/><meta name="docsearch:version" content="24.0.0" /><met [...]
+ window.dataLayer = window.dataLayer || [];
+ function gtag(){dataLayer.push(arguments); }
+ gtag('js', new Date());
+ gtag('config', 'UA-131010415-1');
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ var coll = document.getElementsByClassName('collapsible');
+ var checkActiveCategory = true;
+ for (var i = 0; i < coll.length; i++) {
+ var links = coll[i].nextElementSibling.getElementsByTagName('*');
+ if (checkActiveCategory){
+ for (var j = 0; j < links.length; j++) {
+ if (links[j].classList.contains('navListItemActive')){
+ coll[i].nextElementSibling.classList.toggle('hide');
+ coll[i].childNodes[1].classList.toggle('rotate');
+ checkActiveCategory = false;
+ break;
+ }
+ }
+ }
+
+ coll[i].addEventListener('click', function() {
+ var arrow = this.childNodes[1];
+ arrow.classList.toggle('rotate');
+ var content = this.nextElementSibling;
+ content.classList.toggle('hide');
+ });
+ }
+
+ document.addEventListener('DOMContentLoaded', function() {
+ createToggler('#navToggler', '#docsNav', 'docsSliderActive');
+ createToggler('#tocToggler', 'body', 'tocActive');
+
+ var headings = document.querySelector('.toc-headings');
+ headings && headings.addEventListener('click', function(event) {
+ var el = event.target;
+ while(el !== headings){
+ if (el.tagName === 'A') {
+ document.body.classList.remove('tocActive');
+ break;
+ } else{
+ el = el.parentNode;
+ }
+ }
+ }, false);
+
+ function createToggler(togglerSelector, targetSelector, className) {
+ var toggler = document.querySelector(togglerSelector);
+ var target = document.querySelector(targetSelector);
+
+ if (!toggler) {
+ return;
+ }
+
+ toggler.onclick = function(event) {
+ event.preventDefault();
+
+ target.classList.toggle(className);
+ };
+ }
+ });
+ </script></nav></div><div class="container mainContainer docsContainer"><div class="wrapper"><div class="post"><header class="postHeader"><a class="edit-page-link button" href="https://github.com/apache/druid/edit/master/docs/data-management/update.md" target="_blank" rel="noreferrer noopener">Edit</a><h1 id="__docusaurus" class="postHeaderTitle">Data updates</h1></header><article><div><span><!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<h2><a class="anchor" aria-hidden="true" id="overwrite"></a><a href="#overwrite" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.6 [...]
+<p>Apache Druid stores data <a href="/docs/latest/design/architecture.html#datasources-and-segments">partitioned by time chunk</a> and supports
+overwriting existing data using time ranges. Data outside the replacement time range is not touched. Overwriting of
+existing data is done using the same mechanisms as <a href="/docs/latest/ingestion/index.html#batch">batch ingestion</a>.</p>
+<p>For example:</p>
+<ul>
+<li><a href="/docs/latest/ingestion/native-batch.html">Native batch</a> with <code>appendToExisting: false</code>, and <code>intervals</code> set to a specific
+time range, overwrites data for that time range.</li>
+<li><a href="/docs/latest/multi-stage-query/reference.html#replace">SQL <code>REPLACE <table> OVERWRITE [ALL | WHERE ...]</code></a> overwrites data for
+the entire table or for a specified time range.</li>
+</ul>
+<p>In both cases, Druid's atomic update mechanism ensures that queries will flip seamlessly from the old data to the new
+data on a time-chunk-by-time-chunk basis.</p>
+<p>Ingestion and overwriting cannot run concurrently for the same time range of the same datasource. While an overwrite job
+is ongoing for a particular time range of a datasource, new ingestions for that time range are queued up. Ingestions for
+other time ranges proceed as normal. Read-only queries also proceed as normal, using the pre-existing version of the
+data.</p>
+<p>Druid does not support single-record updates by primary key.</p>
+<h2><a class="anchor" aria-hidden="true" id="reindex"></a><a href="#reindex" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<p>Reindexing is an <a href="#overwrite">overwrite of existing data</a> where the source of new data is the existing data itself. It
+is used to perform schema changes, repartition data, filter out unwanted data, enrich existing data, and so on. This
+behaves just like any other <a href="#overwrite">overwrite</a> with regard to atomic updates and locking.</p>
+<p>With <a href="/docs/latest/ingestion/native-batch.html">native batch</a>, use the <a href="/docs/latest/ingestion/native-batch-input-sources.html#druid-input-source"><code>druid</code> input
+source</a>. If needed,
+<a href="/docs/latest/ingestion/ingestion-spec.html#transformspec"><code>transformSpec</code></a> can be used to filter or modify data during the
+reindexing job.</p>
+<p>With SQL, use <a href="/docs/latest/multi-stage-query/reference.html#replace"><code>REPLACE <table> OVERWRITE</code></a> with `SELECT ... FROM</p>
+<table>`. (Druid does not have `UPDATE` or `ALTER TABLE` statements.) Any SQL SELECT query can be used to filter,
+modify, or enrich the data during the reindexing job.
+<h2><a class="anchor" aria-hidden="true" id="rolled-up-datasources"></a><a href="#rolled-up-datasources" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2 [...]
+<p>Rolled-up datasources can be effectively updated using appends, without rewrites. When you append a row that has an
+identical set of dimensions to an existing row, queries that use aggregation operators automatically combine those two
+rows together at query time.</p>
+<p><a href="/docs/latest/data-management/compaction.html">Compaction</a> or <a href="/docs/latest/data-management/automatic-compaction.html">automatic compaction</a> can be used to physically combine these
+matching rows together later on, by rewriting segments in the background.</p>
+<h2><a class="anchor" aria-hidden="true" id="lookups"></a><a href="#lookups" aria-hidden="true" class="hash-link"><svg class="hash-link-icon" aria-hidden="true" height="16" version="1.1" viewBox="0 0 16 16" width="16"><path fill-rule="evenodd" d="M4 9h1v1H4c-1.5 0-3-1.69-3-3.5S2.55 3 4 3h4c1.45 0 3 1.69 3 3.5 0 1.41-.91 2.72-2 3.25V8.59c.58-.45 1-1.27 1-2.09C10 5.22 8.98 4 8 4H4c-.98 0-2 1.22-2 2.5S3 9 4 9zm9-3h-1v1h1c1 0 2 1.22 2 2.5S13.98 12 13 12H9c-.98 0-2-1.22-2-2.5 0-.83.42-1.64 1- [...]
+<p>If you have a dimension where values need to be updated frequently, try first using <a href="/docs/latest/querying/lookups.html">lookups</a>. A
+classic use case of lookups is when you have an ID dimension stored in a Druid segment, and want to map the ID dimension to a
+human-readable string that may need to be updated periodically.</p>
+</span></div></article></div><div class="docs-prevnext"><a class="docs-prev button" href="/docs/latest/data-management/index.html"><span class="arrow-prev">← </span><span>Overview</span></a><a class="docs-next button" href="/docs/latest/data-management/delete.html"><span>Data deletion</span><span class="arrow-next"> →</span></a></div></div></div><nav class="onPageNav"><ul class="toc-headings"><li><a href="#overwrite">Overwrite</a></li><li><a href="#reindex">Reindex</a></li><li><a href="# [...]
+ document.addEventListener('keyup', function(e) {
+ if (e.target !== document.body) {
+ return;
+ }
+ // keyCode for '/' (slash)
+ if (e.keyCode === 191) {
+ const search = document.getElementById('search_input_react');
+ search && search.focus();
+ }
+ });
+ </script><script>
+ var search = docsearch({
+
+ apiKey: '2de99082a9f38e49dfaa059bbe4c901d',
+ indexName: 'apache_druid',
+ inputSelector: '#search_input_react',
+ algoliaOptions: {"facetFilters":["language:en","version:24.0.0"]}
+ });
+ </script></body></html>
\ No newline at end of file
diff --git a/docs/latest/dependencies/deep-storage.html b/docs/latest/dependencies/deep-storage.html
index 06ab4f9b..d8a3a73b 100644
--- a/docs/latest/dependencies/deep-storage.html
+++ b/docs/latest/dependencies/deep-storage.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/dependencies/metadata-storage.html b/docs/latest/dependencies/metadata-storage.html
index a875830d..e62069ce 100644
--- a/docs/latest/dependencies/metadata-storage.html
+++ b/docs/latest/dependencies/metadata-storage.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/dependencies/zookeeper.html b/docs/latest/dependencies/zookeeper.html
index 6c017bf2..c46197ed 100644
--- a/docs/latest/dependencies/zookeeper.html
+++ b/docs/latest/dependencies/zookeeper.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
diff --git a/docs/latest/design/architecture.html b/docs/latest/design/architecture.html
index af013f8b..d27d0601 100644
--- a/docs/latest/design/architecture.html
+++ b/docs/latest/design/architecture.html
@@ -3,7 +3,7 @@
function gtag(){dataLayer.push(arguments); }
gtag('js', new Date());
gtag('config', 'UA-131010415-1');
- </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
+ </script><link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.7.2/css/all.css"/><link rel="stylesheet" href="/css/code-block-buttons.css"/><script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/clipboard.js/2.0.4/clipboard.min.js"></script><script type="text/javascript" src="/js/code-block-buttons.js"></script><script src="/js/scrollSpy.js"></script><link rel="stylesheet" href="/css/main.css"/><script src="/js/codetabs.js"></script></head>< [...]
var coll = document.getElementsByClassName('collapsible');
var checkActiveCategory = true;
for (var i = 0; i < coll.length; i++) {
@@ -92,8 +92,8 @@
<li><a href="/docs/latest/design/historical.html"><strong>Historical</strong></a> services store queryable data.</li>
<li><a href="/docs/latest/design/middlemanager.html"><strong>MiddleManager</strong></a> services ingest data.</li>
</ul>
... 7358 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org