You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by za...@apache.org on 2026/07/27 12:24:19 UTC
(hive) branch master updated: HIVE-29745: Remove Druid integration (#6615)
This is an automated email from the ASF dual-hosted git repository.
zabetak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e28a1ebdfb5 HIVE-29745: Remove Druid integration (#6615)
e28a1ebdfb5 is described below
commit e28a1ebdfb55288a9b230219343aece6300bbc5a
Author: Stamatis Zampetakis <za...@gmail.com>
AuthorDate: Mon Jul 27 15:24:13 2026 +0300
HIVE-29745: Remove Druid integration (#6615)
Remove the Druid storage handler and all associated infrastructure from the codebase since its broken and no longer used.
Changes include:
- Delete the druid-handler module (storage handler, SerDe, I/O formats, Kafka ingestion support, Kerberos security utilities)
- Delete the itests/qtest-druid module (MiniDruidCluster, test drivers)
- Remove all Druid-related query test files (.q and .q.out)
- Remove hive.druid.* configuration variables from HiveConf
- Remove Druid constants and calcite-druid dependency
- Remove Calcite optimizer rules: HiveDruidRules, DruidSqlOperatorConverter, SortedDynPartitionTimeGranularityOptimizer
- Remove DruidQuery handling from CalcitePlanner, HiveVolcanoPlanner, HiveFilterProjectTransposeRule, HiveRelFieldTrimmer, ASTBuilder, ASTConverter, and HiveMaterializedViewsRegistry
- Remove DRUID/DRUID_LOCAL/DRUID_KAFKA cluster types from test infra
- Relocate Kafka test utilities (SingleNodeKafkaCluster, Wikipedia) from itests/qtest-druid to itests/util under org.apache.hadoop.hive.kafka
- Update itests/util dependencies to use kafka artifacts directly instead of pulling them transitively via hive-it-druid
- Remove/Update obsolete TODOs around Druid
- Remove Druid prefix from metastore constants and regenerate Thrift code
---
.../org/apache/hadoop/hive/conf/Constants.java | 17 -
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 70 -
.../datasets/druid_table_alltypesorc/load.hive.sql | 40 -
data/scripts/q_test_cleanup_druid.sql | 2 -
data/scripts/q_test_druid_init.sql | 0
druid-handler/README.md | 21 -
druid-handler/pom.xml | 434 ----
.../apache/hadoop/hive/druid/DruidKafkaUtils.java | 232 --
.../hadoop/hive/druid/DruidStorageHandler.java | 1020 --------
.../hadoop/hive/druid/DruidStorageHandlerInfo.java | 67 -
.../hive/druid/DruidStorageHandlerUtils.java | 1113 --------
.../hadoop/hive/druid/conf/DruidConstants.java | 81 -
.../hadoop/hive/druid/conf/package-info.java | 23 -
.../hadoop/hive/druid/io/DruidOutputFormat.java | 178 --
.../hive/druid/io/DruidQueryBasedInputFormat.java | 262 --
.../hadoop/hive/druid/io/DruidRecordWriter.java | 300 ---
.../hive/druid/io/DruidVectorizedWrapper.java | 139 -
.../hadoop/hive/druid/io/HiveDruidSplit.java | 80 -
.../apache/hadoop/hive/druid/io/package-info.java | 23 -
.../hadoop/hive/druid/json/AvroBytesDecoder.java | 31 -
.../hadoop/hive/druid/json/AvroParseSpec.java | 83 -
.../hive/druid/json/AvroStreamInputRowParser.java | 76 -
.../druid/json/InlineSchemaAvroBytesDecoder.java | 44 -
.../druid/json/KafkaIndexTaskTuningConfig.java | 128 -
.../hive/druid/json/KafkaSupervisorIOConfig.java | 238 --
.../hive/druid/json/KafkaSupervisorReport.java | 202 --
.../hive/druid/json/KafkaSupervisorSpec.java | 117 -
.../druid/json/KafkaSupervisorTuningConfig.java | 161 --
.../json/SeekableStreamIndexTaskTuningConfig.java | 308 ---
.../json/SeekableStreamSupervisorTuningConfig.java | 59 -
.../hadoop/hive/druid/json/TaskReportData.java | 102 -
.../hadoop/hive/druid/json/package-info.java | 23 -
.../org/apache/hadoop/hive/druid/package-info.java | 23 -
.../hive/druid/security/DruidKerberosUtil.java | 119 -
.../hive/druid/security/KerberosHttpClient.java | 132 -
.../hive/druid/security/ResponseCookieHandler.java | 75 -
.../RetryIfUnauthorizedResponseHandler.java | 101 -
.../hive/druid/security/RetryResponseHolder.java | 45 -
.../hadoop/hive/druid/security/package-info.java | 23 -
.../druid/serde/DruidGroupByQueryRecordReader.java | 93 -
.../hive/druid/serde/DruidQueryRecordReader.java | 303 ---
.../druid/serde/DruidScanQueryRecordReader.java | 88 -
.../apache/hadoop/hive/druid/serde/DruidSerDe.java | 497 ----
.../hadoop/hive/druid/serde/DruidSerDeUtils.java | 64 -
.../serde/DruidTimeseriesQueryRecordReader.java | 89 -
.../druid/serde/DruidTopNQueryRecordReader.java | 96 -
.../hadoop/hive/druid/serde/DruidWritable.java | 131 -
.../hadoop/hive/druid/serde/package-info.java | 23 -
.../hive/druid/DerbyConnectorTestUtility.java | 129 -
...tDruidQueryBasedInputFormatToAddFaultyHost.java | 53 -
.../apache/hadoop/hive/druid/QTestDruidSerDe.java | 104 -
.../apache/hadoop/hive/druid/QTestDruidSerDe2.java | 106 -
.../hive/druid/QTestDruidStorageHandler.java | 34 -
.../hive/druid/QTestDruidStorageHandler2.java | 34 -
.../QTestDruidStorageHandlerToAddFaultyHost.java | 34 -
.../hadoop/hive/druid/TestDruidStorageHandler.java | 774 ------
.../druid/TestHiveDruidQueryBasedInputFormat.java | 177 --
.../hadoop/hive/druid/io/TestHiveDruidSplit.java | 49 -
.../hadoop/hive/druid/serde/TestDruidSerDe.java | 884 -------
.../hadoop/hive/ql/io/TestDruidRecordWriter.java | 245 --
itests/hive-minikdc/pom.xml | 12 -
itests/hive-unit-hadoop2/pom.xml | 6 -
itests/hive-unit/pom.xml | 6 -
.../org/apache/hive/jdbc/TestRestrictedList.java | 2 -
itests/pom.xml | 17 -
itests/qtest-accumulo/pom.xml | 4 -
itests/qtest-druid/pom.xml | 319 ---
.../main/java/org/apache/hive/druid/DruidNode.java | 44 -
.../org/apache/hive/druid/ForkingDruidNode.java | 157 --
.../org/apache/hive/druid/MiniDruidCluster.java | 283 --
itests/qtest-iceberg/pom.xml | 34 -
itests/qtest-kudu/pom.xml | 4 -
itests/qtest/pom.xml | 34 -
.../hive/cli/MiniDruidLlapLocalCliDriver.java | 63 -
.../hadoop/hive/cli/TestMiniDruidCliDriver.java | 64 -
.../hive/cli/TestMiniDruidKafkaCliDriver.java | 64 -
.../test/resources/testconfiguration.properties | 28 -
itests/util/pom.xml | 33 +-
.../apache/hadoop/hive/cli/control/CliConfigs.java | 63 -
.../hadoop}/hive/kafka/SingleNodeKafkaCluster.java | 6 +-
.../org/apache/hadoop}/hive/kafka/Wikipedia.java | 90 +-
.../apache/hadoop/hive/ql/QTestMiniClusters.java | 72 +-
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 2 +-
.../SQLStdHiveAuthorizationValidatorForTest.java | 4 -
.../llap/cli/service/AsyncTaskCopyAuxJars.java | 2 +-
packaging/pom.xml | 5 -
packaging/src/main/assembly/src.xml | 1 -
pom.xml | 9 -
ql/pom.xml | 10 -
.../java/org/apache/hadoop/hive/ql/QueryState.java | 2 -
.../ql/ddl/table/create/CreateTableOperation.java | 1 -
.../ql/metadata/HiveMaterializedViewsRegistry.java | 105 +-
.../apache/hadoop/hive/ql/optimizer/Optimizer.java | 4 -
...SortedDynPartitionTimeGranularityOptimizer.java | 479 ----
.../optimizer/calcite/cost/HiveVolcanoPlanner.java | 23 -
.../ql/optimizer/calcite/rules/HiveDruidRules.java | 304 ---
.../rules/HiveFilterProjectTransposeRule.java | 8 -
.../calcite/rules/HiveRelFieldTrimmer.java | 75 +-
.../rules/views/HiveMaterializedViewUtils.java | 20 +-
.../optimizer/calcite/translator/ASTBuilder.java | 35 +-
.../optimizer/calcite/translator/ASTConverter.java | 14 -
.../calcite/translator/PlanModifierForASTConv.java | 5 -
.../hadoop/hive/ql/parse/CalcitePlanner.java | 190 +-
.../hive/ql/parse/DruidSqlOperatorConverter.java | 392 ---
.../apache/hadoop/hive/ql/util/UpgradeTool.java | 2 +-
.../sqlstd/TestSQLStdHiveAccessControllerHS2.java | 2 +-
ql/src/test/queries/clientnegative/druid_address.q | 5 -
ql/src/test/queries/clientnegative/druid_buckets.q | 6 -
ql/src/test/queries/clientnegative/druid_case.q | 6 -
.../test/queries/clientnegative/druid_datasource.q | 3 -
.../queries/clientnegative/druid_datasource2.q | 4 -
.../test/queries/clientnegative/druid_location.q | 6 -
.../test/queries/clientnegative/druid_partitions.q | 6 -
.../druid_materialized_view_rewrite_ssb.q | 318 ---
.../test/queries/clientpositive/druid_timeseries.q | 111 -
.../queries/clientpositive/druid_timestamptz.q | 94 -
.../queries/clientpositive/druid_timestamptz2.q | 64 -
ql/src/test/queries/clientpositive/druid_topn.q | 75 -
.../queries/clientpositive/druidkafkamini_avro.q | 100 -
.../queries/clientpositive/druidkafkamini_basic.q | 131 -
.../queries/clientpositive/druidkafkamini_csv.q | 38 -
.../clientpositive/druidkafkamini_delimited.q | 39 -
.../clientpositive/druidmini_dynamic_partition.q | 190 --
.../queries/clientpositive/druidmini_expressions.q | 237 --
.../queries/clientpositive/druidmini_extractTime.q | 187 --
.../queries/clientpositive/druidmini_floorTime.q | 148 --
.../test/queries/clientpositive/druidmini_joins.q | 62 -
.../queries/clientpositive/druidmini_masking.q | 26 -
ql/src/test/queries/clientpositive/druidmini_mv.q | 115 -
.../druidmini_semijoin_reduction_all_types.q | 130 -
.../test/queries/clientpositive/druidmini_test1.q | 122 -
.../queries/clientpositive/druidmini_test_alter.q | 56 -
.../queries/clientpositive/druidmini_test_insert.q | 157 --
.../queries/clientpositive/druidmini_test_ts.q | 109 -
.../results/clientnegative/druid_address.q.out | 7 -
.../results/clientnegative/druid_buckets.q.out | 8 -
.../test/results/clientnegative/druid_case.q.out | 7 -
.../results/clientnegative/druid_datasource.q.out | 7 -
.../results/clientnegative/druid_datasource2.q.out | 7 -
.../results/clientnegative/druid_location.q.out | 9 -
.../results/clientnegative/druid_partitions.q.out | 8 -
.../druid_materialized_view_rewrite_ssb.q.out | 813 ------
.../clientpositive/druid/druid_timeseries.q.out | 640 -----
.../clientpositive/druid/druid_timestamptz2.q.out | 209 --
.../results/clientpositive/druid/druid_topn.q.out | 374 ---
.../clientpositive/druid/druidkafkamini_avro.q.out | 263 --
.../druid/druidkafkamini_basic.q.out | 767 ------
.../clientpositive/druid/druidkafkamini_csv.q.out | 138 -
.../druid/druidkafkamini_delimited.q.out | 140 -
.../druid/druidmini_dynamic_partition.q.out | 682 -----
.../druid/druidmini_expressions.q.out | 2737 --------------------
.../druid/druidmini_extractTime.q.out | 886 -------
.../clientpositive/druid/druidmini_floorTime.q.out | 692 -----
.../clientpositive/druid/druidmini_mv.q.out | 741 ------
.../druidmini_semijoin_reduction_all_types.q.out | 2033 ---------------
.../clientpositive/druid/druidmini_test1.q.out | 960 -------
.../druid/druidmini_test_alter.q.out | 210 --
.../druid/druidmini_test_insert.q.out | 875 -------
.../clientpositive/druid/druidmini_test_ts.q.out | 556 ----
.../thrift/gen-cpp/hive_metastore_constants.cpp | 2 -
.../gen/thrift/gen-cpp/hive_metastore_constants.h | 1 -
.../metastore/api/hive_metastoreConstants.java | 2 -
.../src/gen/thrift/gen-php/metastore/Constant.php | 6 -
.../gen/thrift/gen-py/hive_metastore/constants.py | 1 -
.../gen/thrift/gen-rb/hive_metastore_constants.rb | 2 -
.../hive/metastore/utils/MetaStoreUtils.java | 3 +-
.../src/main/thrift/hive_metastore.thrift | 1 -
167 files changed, 142 insertions(+), 29488 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
index 4b0e75fafb5..7aa6945c688 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java
@@ -30,23 +30,6 @@ public class Constants {
public static final String LLAP_NUM_BUCKETS = "llap.num.buckets";
public static final String LLAP_BUCKET_ID = "llap.bucket.id";
- /* Constants for Druid storage handler */
- public static final String DRUID_HIVE_STORAGE_HANDLER_ID =
- "org.apache.hadoop.hive.druid.DruidStorageHandler";
- public static final String DRUID_HIVE_OUTPUT_FORMAT =
- "org.apache.hadoop.hive.druid.io.DruidOutputFormat";
- public static final String DRUID_DATA_SOURCE = "druid.datasource";
- public static final String DRUID_SEGMENT_GRANULARITY = "druid.segment.granularity";
-
- public static final String DRUID_TARGET_SHARDS_PER_GRANULARITY =
- "druid.segment.targetShardsPerGranularity";
- public static final String DRUID_TIMESTAMP_GRANULARITY_COL_NAME = "__time_granularity";
- public static final String DRUID_SHARD_KEY_COL_NAME = "__druid_extra_partition_key";
- public static final String DRUID_QUERY_JSON = "druid.query.json";
- public static final String DRUID_QUERY_FIELD_NAMES = "druid.fieldNames";
- public static final String DRUID_QUERY_FIELD_TYPES = "druid.fieldTypes";
- public static final String DRUID_QUERY_TYPE = "druid.query.type";
-
public static final String JDBC_HIVE_STORAGE_HANDLER_ID =
"org.apache.hive.storage.jdbc.JdbcStorageHandler";
public static final String JDBC_CONFIG_PREFIX = "hive.sql";
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 65ad8a82bbe..54516007d32 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -3382,72 +3382,6 @@ public static enum ConfVars {
OPTIMIZE_ACID_META_COLUMNS("hive.optimize.acid.meta.columns", true,
"If true, don't decode Acid metadata columns from storage unless" +
" they are needed."),
-
- // For Druid storage handler
- HIVE_DRUID_INDEXING_GRANULARITY("hive.druid.indexer.segments.granularity", "DAY",
- new PatternSet("YEAR", "MONTH", "WEEK", "DAY", "HOUR", "MINUTE", "SECOND"),
- "Granularity for the segments created by the Druid storage handler"
- ),
- HIVE_DRUID_MAX_PARTITION_SIZE("hive.druid.indexer.partition.size.max", 5000000,
- "Maximum number of records per segment partition"
- ),
- HIVE_DRUID_MAX_ROW_IN_MEMORY("hive.druid.indexer.memory.rownum.max", 75000,
- "Maximum number of records in memory while storing data in Druid"
- ),
- HIVE_DRUID_BROKER_DEFAULT_ADDRESS("hive.druid.broker.address.default", "localhost:8082",
- "Address of the Druid broker. If we are querying Druid from Hive, this address needs to be\n"
- +
- "declared"
- ),
- HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS("hive.druid.coordinator.address.default", "localhost:8081",
- "Address of the Druid coordinator. It is used to check the load status of newly created segments"
- ),
- HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS("hive.druid.overlord.address.default", "localhost:8090",
- "Address of the Druid overlord. It is used to submit indexing tasks to druid."
- ),
- HIVE_DRUID_SELECT_THRESHOLD("hive.druid.select.threshold", 10000,
- "Takes only effect when hive.druid.select.distribute is set to false. \n" +
- "When we can split a Select query, this is the maximum number of rows that we try to retrieve\n" +
- "per query. In order to do that, we obtain the estimated size for the complete result. If the\n" +
- "number of records of the query results is larger than this threshold, we split the query in\n" +
- "total number of rows/threshold parts across the time dimension. Note that we assume the\n" +
- "records to be split uniformly across the time dimension."),
- HIVE_DRUID_NUM_HTTP_CONNECTION("hive.druid.http.numConnection", 20, "Number of connections used by\n" +
- "the HTTP client."),
- HIVE_DRUID_HTTP_READ_TIMEOUT("hive.druid.http.read.timeout", "PT1M", "Read timeout period for the HTTP\n" +
- "client in ISO8601 format (for example P2W, P3M, PT1H30M, PT0.750S), default is period of 1 minute."),
- HIVE_DRUID_SLEEP_TIME("hive.druid.sleep.time", "PT10S",
- "Sleep time between retries in ISO8601 format (for example P2W, P3M, PT1H30M, PT0.750S), default is period of 10 seconds."
- ),
- HIVE_DRUID_BASE_PERSIST_DIRECTORY("hive.druid.basePersistDirectory", "",
- "Local temporary directory used to persist intermediate indexing state, will default to JVM system property java.io.tmpdir."
- ),
- HIVE_DRUID_ROLLUP("hive.druid.rollup", true, "Whether to rollup druid rows or not."),
- DRUID_SEGMENT_DIRECTORY("hive.druid.storage.storageDirectory", "/druid/segments"
- , "druid deep storage location."),
- DRUID_METADATA_BASE("hive.druid.metadata.base", "druid", "Default prefix for metadata tables"),
- DRUID_METADATA_DB_TYPE("hive.druid.metadata.db.type", "mysql",
- new PatternSet("mysql", "postgresql", "derby"), "Type of the metadata database."
- ),
- DRUID_METADATA_DB_USERNAME("hive.druid.metadata.username", "",
- "Username to connect to Type of the metadata DB."
- ),
- DRUID_METADATA_DB_PASSWORD("hive.druid.metadata.password", "",
- "Password to connect to Type of the metadata DB."
- ),
- DRUID_METADATA_DB_URI("hive.druid.metadata.uri", "",
- "URI to connect to the database (for example jdbc:mysql://hostname:port/DBName)."
- ),
- DRUID_WORKING_DIR("hive.druid.working.directory", "/tmp/workingDirectory",
- "Default hdfs working directory used to store some intermediate metadata"
- ),
- HIVE_DRUID_MAX_TRIES("hive.druid.maxTries", 5, "Maximum number of retries before giving up"),
- HIVE_DRUID_PASSIVE_WAIT_TIME("hive.druid.passiveWaitTimeMs", 30000L,
- "Wait time in ms default to 30 seconds."
- ),
- HIVE_DRUID_BITMAP_FACTORY_TYPE("hive.druid.bitmap.type", "roaring", new PatternSet("roaring", "concise"), "Coding algorithm use to encode the bitmaps"),
- HIVE_DRUID_KERBEROS_ENABLE("hive.druid.kerberos.enable", true,
- "Enable/Disable Kerberos authentication explicitly while connecting to a druid cluster."),
// For HBase storage handler
HIVE_HBASE_WAL_ENABLED("hive.hbase.wal.enabled", true,
"Whether writes to HBase should be forced to the write-ahead log. \n" +
@@ -5570,8 +5504,6 @@ public static enum ConfVars {
"hive.privilege.synchronizer," +
"hive.privilege.synchronizer.interval," +
"hive.query.max.length," +
- "hive.druid.broker.address.default," +
- "hive.druid.coordinator.address.default," +
"hikaricp.," +
"hadoop.bin.path," +
"yarn.bin.path," +
@@ -5591,7 +5523,6 @@ public static enum ConfVars {
HIVE_CONF_HIDDEN_LIST("hive.conf.hidden.list",
METASTORE_PWD.varname + "," + HIVE_SERVER2_SSL_KEYSTORE_PASSWORD.varname
+ "," + HIVE_SERVER2_WEBUI_SSL_KEYSTORE_PASSWORD.varname
- + "," + DRUID_METADATA_DB_PASSWORD.varname
// Adding the S3 credentials from Hadoop config to be hidden
+ ",fs.s3.awsAccessKeyId"
+ ",fs.s3.awsSecretAccessKey"
@@ -6787,7 +6718,6 @@ private static String[] convertVarsToRegex(String[] paramList) {
"hive\\.auto\\..*",
"hive\\.cbo\\..*",
"hive\\.convert\\..*",
- "hive\\.druid\\..*",
"hive\\.exec\\.dynamic\\.partition.*",
"hive\\.exec\\.max\\.dynamic\\.partitions.*",
"hive\\.exec\\.compress\\..*",
diff --git a/data/files/datasets/druid_table_alltypesorc/load.hive.sql b/data/files/datasets/druid_table_alltypesorc/load.hive.sql
deleted file mode 100644
index c322e3fd3f4..00000000000
--- a/data/files/datasets/druid_table_alltypesorc/load.hive.sql
+++ /dev/null
@@ -1,40 +0,0 @@
-CREATE TABLE alltypesorc1(
- ctinyint TINYINT,
- csmallint SMALLINT,
- cint INT,
- cbigint BIGINT,
- cfloat FLOAT,
- cdouble DOUBLE,
- cstring1 STRING,
- cstring2 STRING,
- ctimestamp1 TIMESTAMP,
- ctimestamp2 TIMESTAMP,
- cboolean1 BOOLEAN,
- cboolean2 BOOLEAN,
- cintstring STRING,
- cfloatstring STRING,
- cdoublestring STRING)
- STORED AS ORC;
-
-LOAD DATA LOCAL INPATH "${hiveconf:test.data.dir}/alltypesorc"
-OVERWRITE INTO TABLE alltypesorc1;
-
-CREATE EXTERNAL TABLE druid_table_alltypesorc
-STORED BY 'org.apache.hadoop.hive.druid.DruidStorageHandler'
-TBLPROPERTIES ("druid.segment.granularity" = "HOUR", "druid.query.granularity" = "MINUTE")
-AS
-SELECT cast (`ctimestamp1` as timestamp with local time zone) as `__time`,
- cstring1,
- cstring2,
- cdouble,
- cfloat,
- ctinyint,
- csmallint,
- cint,
- cbigint,
- cboolean1,
- cboolean2,
- cast(cint as string) as cintstring,
- cast(cfloat as string) as cfloatstring,
- cast(cdouble as string) as cdoublestring
- FROM alltypesorc1 where ctimestamp1 IS NOT NULL;
diff --git a/data/scripts/q_test_cleanup_druid.sql b/data/scripts/q_test_cleanup_druid.sql
deleted file mode 100644
index b0d34257047..00000000000
--- a/data/scripts/q_test_cleanup_druid.sql
+++ /dev/null
@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS alltypesorc;
-DROP TABLE IF EXISTS druid_table;
diff --git a/data/scripts/q_test_druid_init.sql b/data/scripts/q_test_druid_init.sql
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/druid-handler/README.md b/druid-handler/README.md
deleted file mode 100644
index 285f7ec162a..00000000000
--- a/druid-handler/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
-{% comment %}
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to you under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-{% endcomment %}
--->
-# Druid Storage Handler
-
-[Link for documentation](https://hive.apache.org/docs/latest/user/druid-integration)
diff --git a/druid-handler/pom.xml b/druid-handler/pom.xml
deleted file mode 100644
index dcc5ebd8a48..00000000000
--- a/druid-handler/pom.xml
+++ /dev/null
@@ -1,434 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive</artifactId>
- <version>4.3.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <artifactId>hive-druid-handler</artifactId>
- <packaging>jar</packaging>
- <name>Hive Druid Handler</name>
- <properties>
- <hive.path.to.root>..</hive.path.to.root>
- <druid.guava.version>16.0.1</druid.guava.version>
- </properties>
- <dependencies>
- <!-- dependencies are always listed in sorted order by groupId, artifactId -->
- <!-- intra-project -->
- <dependency>
- <groupId>com.fasterxml.jackson.dataformat</groupId>
- <artifactId>jackson-dataformat-smile</artifactId>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </dependency>
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- <version>${validation-api.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive-exec</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- inter-project -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>${druid.guava.version}</version>
- </dependency>
- <dependency>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty</artifactId>
- <version>${netty3.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-server</artifactId>
- <version>${druid.version}</version>
- <exclusions>
- <exclusion>
- <groupId>io.tesla.aether</groupId>
- <artifactId>tesla-aether</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-aws-common</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.glassfish</groupId>
- <artifactId>javax.el</artifactId>
- </exclusion>
- </exclusions>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-processing</artifactId>
- <version>${druid.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.ibm.icu</groupId>
- <artifactId>icu4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.codehaus.plexus</groupId>
- <artifactId>plexus-utils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm-commons</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.ow2.asm</groupId>
- <artifactId>asm</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.checkerframework</groupId>
- <artifactId>checker-qual</artifactId>
- </exclusion>
- <exclusion>
- <!--LGPL licenced library-->
- <groupId>com.google.code.findbugs</groupId>
- <artifactId>annotations</artifactId>
- </exclusion>
- </exclusions>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-hdfs-storage</artifactId>
- <version>${druid.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-aws</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- </exclusion>
- </exclusions>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>mysql-metadata-storage</artifactId>
- <version>${druid.version}</version>
- <exclusions>
- <exclusion>
- <groupId>mysql</groupId>
- <artifactId>mysql-connector-java</artifactId>
- </exclusion>
- </exclusions>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>postgresql-metadata-storage</artifactId>
- <version>${druid.version}</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.druid.extensions</groupId>
- <artifactId>druid-bloom-filter</artifactId>
- <version>${druid.version}</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-common</artifactId>
- <scope>provided</scope>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>ch.qos.reload4j</groupId>
- <artifactId>reload4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-beanutils</groupId>
- <artifactId>commons-beanutils</artifactId>
- </exclusion>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <scope>provided</scope>
- <artifactId>hadoop-mapreduce-client-core</artifactId>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.inject.extensions</groupId>
- <artifactId>guice-servlet</artifactId>
- </exclusion>
- <exclusion>
- <groupId>io.netty</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!-- test inter-project -->
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>${mockito-core.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.druid</groupId>
- <artifactId>druid-indexing-hadoop</artifactId>
- <version>${druid.version}</version>
- <scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-api</artifactId>
- <version>${log4j2.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <profiles>
- <profile>
- <id>dev-fast-build</id>
- <activation>
- <property>
- <name>skipShade</name>
- <value>!true</value>
- </property>
- </activation>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <!-- we need to shade netty, as there is a conflict between versions
- used by Hadoop (3.6.2.Final) and Druid (3.10.4.Final) -->
- <!-- we need to shade jackson, as there is a conflict between versions
- used by Hive (2.4.2) and Druid (2.4.6) -->
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadeTestJar>true</shadeTestJar>
- <createDependencyReducedPom>false</createDependencyReducedPom>
- <relocations>
- <relocation>
- <pattern>org.apache.druid</pattern>
- <shadedPattern>org.apache.hive.druid.org.apache.druid</shadedPattern>
- </relocation>
- <relocation>
- <pattern>io.netty</pattern>
- <shadedPattern>org.apache.hive.druid.io.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.apache.calcite</pattern>
- <shadedPattern>org.apache.hive.druid.org.apache.calcite</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.jboss.netty</pattern>
- <shadedPattern>org.apache.hive.druid.org.jboss.netty</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.fasterxml.jackson</pattern>
- <shadedPattern>org.apache.hive.druid.com.fasterxml.jackson</shadedPattern>
- </relocation>
- <relocation>
- <pattern>com.google.common</pattern>
- <shadedPattern>org.apache.hive.druid.com.google.common</shadedPattern>
- </relocation>
- <relocation>
- <pattern>org.roaringbitmap</pattern>
- <shadedPattern>org.apache.hive.druid.org.roaringbitmap</shadedPattern>
- </relocation>
- </relocations>
- <artifactSet>
- <includes>
- <include>org.apache.druid:*</include>
- <include>org.apache.druid.extensions:*</include>
- <include>io.netty:*</include>
- <include>org.apache.calcite:*</include>
- <include>com.fasterxml.jackson.core:*</include>
- <include>com.fasterxml.jackson.datatype:*</include>
- <include>com.fasterxml.jackson.dataformat:*</include>
- <include>com.google.guava:*</include>
- <include>it.unimi.dsi:*</include>
- <include>org.jdbi:*</include>
- <include>net.jpountz.lz4:*</include>
- <include>org.apache.commons:*</include>
- <include>org.roaringbitmap:*</include>
- <include>org.apache.derby:*</include>
- <include>org.asynchttpclient:*</include>
- <include>org.antlr:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- <exclude>META-INF/versions/**</exclude>
- <exclude>static/</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- </profiles>
- <build>
- <sourceDirectory>${basedir}/src/java</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test</testSourceDirectory>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java
deleted file mode 100644
index fb6ce308fbc..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidKafkaUtils.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableMap;
-import org.apache.druid.data.input.impl.CSVParseSpec;
-import org.apache.druid.data.input.impl.DelimitedParseSpec;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.InputRowParser;
-import org.apache.druid.data.input.impl.JSONParseSpec;
-import org.apache.druid.data.input.impl.StringInputRowParser;
-import org.apache.druid.data.input.impl.TimestampSpec;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHandler;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHolder;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.druid.json.AvroParseSpec;
-import org.apache.hadoop.hive.druid.json.AvroStreamInputRowParser;
-import org.apache.hadoop.hive.druid.json.InlineSchemaAvroBytesDecoder;
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorIOConfig;
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorSpec;
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorTuningConfig;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URL;
-import java.nio.charset.Charset;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.hadoop.hive.druid.DruidStorageHandlerUtils.JSON_MAPPER;
-
-/**
- * Class containing some Utility methods for Kafka Ingestion.
- */
-final class DruidKafkaUtils {
- private static final Logger LOG = LoggerFactory.getLogger(DruidKafkaUtils.class);
- private static final SessionState.LogHelper CONSOLE = new SessionState.LogHelper(LOG);
-
- private DruidKafkaUtils() {
- }
-
- static KafkaSupervisorSpec createKafkaSupervisorSpec(
- Table table,
- String kafkaTopic,
- String kafkaServers,
- DataSchema dataSchema,
- IndexSpec indexSpec
- ) {
- return new KafkaSupervisorSpec(dataSchema,
- new KafkaSupervisorTuningConfig(DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxRowsInMemory"),
- DruidStorageHandlerUtils.getLongProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxBytesInMemory"),
-
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxRowsPerSegment"), DruidStorageHandlerUtils
- .getLongProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxTotalRows"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "intermediatePersistPeriod"), null,
- // basePersistDirectory - use druid default, no need to be configured by user
- DruidStorageHandlerUtils
- .getIntegerProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxPendingPersists"),
- indexSpec, null, null,
- // buildV9Directly - use druid default, no need to be configured by user
- DruidStorageHandlerUtils.getBooleanProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "reportParseExceptions"),
- DruidStorageHandlerUtils.getLongProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "handoffConditionTimeout"),
- DruidStorageHandlerUtils.getBooleanProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "resetOffsetAutomatically"),
- TmpFileSegmentWriteOutMediumFactory.instance(), DruidStorageHandlerUtils
- .getIntegerProperty(table, DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "workerThreads"),
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "chatThreads"),
- DruidStorageHandlerUtils.getLongProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "chatRetries"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "httpTimeout"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "shutdownTimeout"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "offsetFetchPeriod"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "intermediateHandoffPeriod"),
- DruidStorageHandlerUtils.getBooleanProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "logParseExceptions"),
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxParseExceptions"),
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "maxSavedParseExceptions")),
- new KafkaSupervisorIOConfig(kafkaTopic,
- // Mandatory Property
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "replicas"),
- DruidStorageHandlerUtils.getIntegerProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "taskCount"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "taskDuration"),
- getKafkaConsumerProperties(table, kafkaServers),
- // Mandatory Property
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "startDelay"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "period"),
- DruidStorageHandlerUtils.getBooleanProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "useEarliestOffset"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "completionTimeout"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "lateMessageRejectionPeriod"),
- DruidStorageHandlerUtils.getPeriodProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "earlyMessageRejectionPeriod"),
- DruidStorageHandlerUtils.getBooleanProperty(table,
- DruidConstants.DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "skipOffsetGaps")),
- new HashMap<>());
- }
-
- private static Map<String, String> getKafkaConsumerProperties(Table table, String kafkaServers) {
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- builder.put(KafkaSupervisorIOConfig.BOOTSTRAP_SERVERS_KEY, kafkaServers);
- for (Map.Entry<String, String> entry : table.getParameters().entrySet()) {
- if (entry.getKey().startsWith(DruidConstants.DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX)) {
- String propertyName = entry.getKey().substring(DruidConstants.DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX.length());
- builder.put(propertyName, entry.getValue());
- }
- }
- return builder.build();
- }
-
- static void updateKafkaIngestionSpec(String overlordAddress, KafkaSupervisorSpec spec) {
- try {
- String task = JSON_MAPPER.writeValueAsString(spec);
- CONSOLE.printInfo("submitting kafka Spec {}", task);
- LOG.info("submitting kafka Supervisor Spec {}", task);
- StringFullResponseHolder response = DruidStorageHandlerUtils
- .getResponseFromCurrentLeader(DruidStorageHandler.getHttpClient(), new Request(HttpMethod.POST,
- new URL(String.format("http://%s/druid/indexer/v1/supervisor", overlordAddress)))
- .setContent("application/json", JSON_MAPPER.writeValueAsBytes(spec)),
- new StringFullResponseHandler(Charset.forName("UTF-8")));
- if (response.getStatus().equals(HttpResponseStatus.OK)) {
- String
- msg =
- String.format("Kafka Supervisor for [%s] Submitted Successfully to druid.",
- spec.getDataSchema().getDataSource());
- LOG.info(msg);
- CONSOLE.printInfo(msg);
- } else {
- throw new IOException(String.format("Unable to update Kafka Ingestion for Druid status [%d] full response [%s]",
- response.getStatus().getCode(),
- response.getContent()));
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- static boolean isKafkaStreamingTable(Table table) {
- // For kafka Streaming tables it is mandatory to set a kafka topic.
- return DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.KAFKA_TOPIC) != null;
- }
-
- static InputRowParser getInputRowParser(Table table, TimestampSpec timestampSpec, DimensionsSpec dimensionsSpec) {
- String parseSpecFormat = DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_FORMAT);
-
- // Default case JSON
- if ((parseSpecFormat == null) || "json".equalsIgnoreCase(parseSpecFormat)) {
- return new StringInputRowParser(new JSONParseSpec(timestampSpec, dimensionsSpec, null, null), "UTF-8");
- } else if ("csv".equalsIgnoreCase(parseSpecFormat)) {
- return new StringInputRowParser(new CSVParseSpec(timestampSpec,
- dimensionsSpec,
- DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_LIST_DELIMITER),
- DruidStorageHandlerUtils.getListProperty(table, DruidConstants.DRUID_PARSE_SPEC_COLUMNS),
- DruidStorageHandlerUtils.getBooleanProperty(table, DruidConstants.DRUID_PARSE_SPEC_HAS_HEADER_ROWS, false),
- DruidStorageHandlerUtils.getIntegerProperty(table, DruidConstants.DRUID_PARSE_SPEC_SKIP_HEADER_ROWS, 0)),
- "UTF-8");
- } else if ("delimited".equalsIgnoreCase(parseSpecFormat)) {
- return new StringInputRowParser(new DelimitedParseSpec(timestampSpec,
- dimensionsSpec,
- DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_DELIMITER),
- DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_PARSE_SPEC_LIST_DELIMITER),
- DruidStorageHandlerUtils.getListProperty(table, DruidConstants.DRUID_PARSE_SPEC_COLUMNS),
- DruidStorageHandlerUtils.getBooleanProperty(table, DruidConstants.DRUID_PARSE_SPEC_HAS_HEADER_ROWS, false),
- DruidStorageHandlerUtils.getIntegerProperty(table, DruidConstants.DRUID_PARSE_SPEC_SKIP_HEADER_ROWS, 0)),
- "UTF-8");
- } else if ("avro".equalsIgnoreCase(parseSpecFormat)) {
- try {
- String avroSchemaLiteral = DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.AVRO_SCHEMA_LITERAL);
- Preconditions.checkNotNull(avroSchemaLiteral, "Please specify avro schema literal when using avro parser");
- Map<String, Object>
- avroSchema =
- JSON_MAPPER.readValue(avroSchemaLiteral, new TypeReference<Map<String, Object>>() {
- });
- return new AvroStreamInputRowParser(new AvroParseSpec(timestampSpec, dimensionsSpec, null),
- new InlineSchemaAvroBytesDecoder(avroSchema));
- } catch (Exception e) {
- throw new IllegalStateException("Exception while creating avro schema", e);
- }
- }
-
- throw new IllegalArgumentException("Invalid parse spec format ["
- + parseSpecFormat
- + "]. "
- + "Supported types are : json, csv, tsv, avro");
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
deleted file mode 100644
index 656fa40c03f..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandler.java
+++ /dev/null
@@ -1,1020 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.base.Supplier;
-import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.InputRowParser;
-import org.apache.druid.data.input.impl.TimestampSpec;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.RetryUtils;
-import org.apache.druid.java.util.common.lifecycle.Lifecycle;
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.java.util.http.client.HttpClientConfig;
-import org.apache.druid.java.util.http.client.HttpClientInit;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHandler;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHolder;
-import org.apache.druid.metadata.MetadataStorageConnectorConfig;
-import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.SQLMetadataConnector;
-import org.apache.druid.metadata.storage.derby.DerbyConnector;
-import org.apache.druid.metadata.storage.derby.DerbyMetadataStorage;
-import org.apache.druid.metadata.storage.mysql.MySQLConnector;
-import org.apache.druid.metadata.storage.mysql.MySQLConnectorConfig;
-import org.apache.druid.metadata.storage.postgresql.PostgreSQLConnector;
-import org.apache.druid.metadata.storage.postgresql.PostgreSQLConnectorConfig;
-import org.apache.druid.metadata.storage.postgresql.PostgreSQLTablesConfig;
-import org.apache.druid.query.BaseQuery;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.indexing.granularity.GranularitySpec;
-import org.apache.druid.segment.loading.DataSegmentPusher;
-import org.apache.druid.segment.loading.SegmentLoadingException;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.TableName;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.druid.io.DruidOutputFormat;
-import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat;
-import org.apache.hadoop.hive.druid.io.DruidRecordWriter;
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorReport;
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorSpec;
-import org.apache.hadoop.hive.druid.security.KerberosHttpClient;
-import org.apache.hadoop.hive.druid.serde.DruidSerDe;
-import org.apache.hadoop.hive.metastore.DefaultHiveMetaHook;
-import org.apache.hadoop.hive.metastore.HiveMetaHook;
-import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
-import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.LockType;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.hooks.WriteEntity;
-import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
-import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.TableDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.hive.ql.security.authorization.DefaultHiveAuthorizationProvider;
-import org.apache.hadoop.hive.ql.security.authorization.HiveAuthorizationProvider;
-import org.apache.hadoop.hive.ql.security.authorization.HiveCustomStorageHandlerUtils;
-import org.apache.hadoop.hive.ql.session.SessionState;
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.mapred.InputFormat;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.OutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hive.common.util.ShutdownHookManager;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.joda.time.DateTime;
-import org.joda.time.Period;
-import org.skife.jdbi.v2.exceptions.CallbackFailedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hive.druid.DruidStorageHandlerUtils.JSON_MAPPER;
-
-/**
- * DruidStorageHandler provides a HiveStorageHandler implementation for Druid.
- */
-@SuppressWarnings({ "rawtypes" }) public class DruidStorageHandler extends DefaultHiveMetaHook
- implements HiveStorageHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidStorageHandler.class);
-
- private static final SessionState.LogHelper CONSOLE = new SessionState.LogHelper(LOG);
-
- public static final String SEGMENTS_DESCRIPTOR_DIR_NAME = "segmentsDescriptorDir";
-
- private static final String INTERMEDIATE_SEGMENT_DIR_NAME = "intermediateSegmentDir";
-
- private static final HttpClient HTTP_CLIENT;
-
- private static final List<String> ALLOWED_ALTER_TYPES =
- ImmutableList.of("ADDPROPS", "DROPPROPS", "ADDCOLS");
-
- /** Druid prefix to form the URI for authentication */
- private static final String DRUID_PREFIX = "druid:";
- /** Druid config for determining the host name */
- private static final String DRUID_HOST_NAME = "druid.zk.service.host";
-
- static {
- final Lifecycle lifecycle = new Lifecycle();
- try {
- lifecycle.start();
- } catch (Exception e) {
- LOG.error("Issues with lifecycle start", e);
- }
- HTTP_CLIENT = makeHttpClient(lifecycle);
- ShutdownHookManager.addShutdownHook(lifecycle::stop);
- }
-
- private SQLMetadataConnector connector;
-
- private MetadataStorageTablesConfig druidMetadataStorageTablesConfig = null;
-
- private String uniqueId = null;
-
- private String rootWorkingDir = null;
-
- private Configuration conf;
-
- public DruidStorageHandler() {
- }
-
- @VisibleForTesting public DruidStorageHandler(SQLMetadataConnector connector,
- MetadataStorageTablesConfig druidMetadataStorageTablesConfig) {
- this.connector = connector;
- this.druidMetadataStorageTablesConfig = druidMetadataStorageTablesConfig;
- }
-
- @Override public Class<? extends InputFormat> getInputFormatClass() {
- return DruidQueryBasedInputFormat.class;
- }
-
- @Override public Class<? extends OutputFormat> getOutputFormatClass() {
- return DruidOutputFormat.class;
- }
-
- @Override public Class<? extends AbstractSerDe> getSerDeClass() {
- return DruidSerDe.class;
- }
-
- @Override public HiveMetaHook getMetaHook() {
- return this;
- }
-
- @Override public HiveAuthorizationProvider getAuthorizationProvider() {
- return new DefaultHiveAuthorizationProvider();
- }
-
- @Override public void configureInputJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) {
-
- }
-
- @Override public void configureInputJobCredentials(TableDesc tableDesc, Map<String, String> jobSecrets) {
-
- }
-
- @Override public void preCreateTable(Table table) throws MetaException {
- if (!StringUtils.isEmpty(table.getSd().getLocation())) {
- throw new MetaException("LOCATION may not be specified for Druid");
- }
- if (table.getPartitionKeysSize() != 0) {
- throw new MetaException("PARTITIONED BY may not be specified for Druid");
- }
- if (table.getSd().getBucketColsSize() != 0) {
- throw new MetaException("CLUSTERED BY may not be specified for Druid");
- }
- String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
- if (dataSourceName != null) {
- // Already Existing datasource in Druid.
- return;
- }
-
- // create dataSourceName based on Hive Table name
- dataSourceName = TableName.getDbTable(table.getDbName(), table.getTableName());
- try {
- // NOTE: This just created druid_segments table in Druid metastore.
- // This is needed for the case when hive is started before any of druid services
- // and druid_segments table has not been created yet.
- getConnector().createSegmentTable();
- } catch (Exception e) {
- LOG.error("Exception while trying to create druid segments table", e);
- throw new MetaException(e.getMessage());
- }
- Collection<String>
- existingDataSources =
- DruidStorageHandlerUtils.getAllDataSourceNames(getConnector(), getDruidMetadataStorageTablesConfig());
- LOG.debug("pre-create data source with name {}", dataSourceName);
- // Check for existence of for the datasource we are going to create in druid_segments table.
- if (existingDataSources.contains(dataSourceName)) {
- throw new MetaException(String.format("Data source [%s] already existing", dataSourceName));
- }
- table.getParameters().put(Constants.DRUID_DATA_SOURCE, dataSourceName);
- }
-
- @Override public void rollbackCreateTable(Table table) {
- cleanWorkingDir();
- }
-
- @Override public void commitCreateTable(Table table) throws MetaException {
- if (DruidKafkaUtils.isKafkaStreamingTable(table)) {
- updateKafkaIngestion(table);
- }
- // For CTAS queries when user has explicitly specified the datasource.
- // We will append the data to existing druid datasource.
- this.commitInsertTable(table, false);
- }
-
- @Override
- public URI getURIForAuth(Table table) throws URISyntaxException{
- Map<String, String> tableProperties = HiveCustomStorageHandlerUtils.getTableProperties(table);
- String host_name = conf.get(DRUID_HOST_NAME) != null ? conf.get(DRUID_HOST_NAME) :
- HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
- String table_name = tableProperties.get(Constants.DRUID_DATA_SOURCE);
- String column_names = tableProperties.get(Constants.DRUID_QUERY_FIELD_NAMES);
- if (column_names != null)
- return new URI(DRUID_PREFIX+"//"+host_name+"/"+table_name+"/"+column_names);
- else
- return new URI(DRUID_PREFIX+"//"+host_name+"/"+table_name);
- }
-
- private void updateKafkaIngestion(Table table) {
- final String overlordAddress = HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS);
-
- final String
- dataSourceName =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE),
- "Druid datasource name is null");
-
- final String
- kafkaTopic =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table,
- DruidConstants.KAFKA_TOPIC), "kafka topic is null");
- final String
- kafkaServers =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table,
- DruidConstants.KAFKA_BOOTSTRAP_SERVERS), "kafka connect string is null");
-
- Properties tableProperties = new Properties();
- tableProperties.putAll(table.getParameters());
-
- final GranularitySpec granularitySpec = DruidStorageHandlerUtils.getGranularitySpec(getConf(), tableProperties);
-
- List<FieldSchema> columns = table.getSd().getCols();
- List<String> columnNames = new ArrayList<>(columns.size());
- List<TypeInfo> columnTypes = new ArrayList<>(columns.size());
-
- for (FieldSchema schema : columns) {
- columnNames.add(schema.getName());
- columnTypes.add(TypeInfoUtils.getTypeInfoFromTypeString(schema.getType()));
- }
-
- Pair<List<DimensionSchema>, AggregatorFactory[]>
- dimensionsAndAggregates =
- DruidStorageHandlerUtils.getDimensionsAndAggregates(columnNames, columnTypes);
- if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- throw new IllegalStateException("Timestamp column (' "
- + DruidConstants.DEFAULT_TIMESTAMP_COLUMN
- + "') not specified in create table; list of columns is : "
- + columnNames);
- }
-
- DimensionsSpec dimensionsSpec = new DimensionsSpec(dimensionsAndAggregates.lhs, null, null);
- String timestampFormat = DruidStorageHandlerUtils
- .getTableProperty(table, DruidConstants.DRUID_TIMESTAMP_FORMAT);
- String timestampColumnName = DruidStorageHandlerUtils
- .getTableProperty(table, DruidConstants.DRUID_TIMESTAMP_COLUMN);
- if(timestampColumnName == null) {
- timestampColumnName = DruidConstants.DEFAULT_TIMESTAMP_COLUMN;
- }
- final TimestampSpec timestampSpec = new TimestampSpec(timestampColumnName, timestampFormat,
- null
- );
- final InputRowParser inputRowParser = DruidKafkaUtils
- .getInputRowParser(table, timestampSpec, dimensionsSpec);
-
- final Map<String, Object>
- inputParser =
- JSON_MAPPER.convertValue(inputRowParser, new TypeReference<Map<String, Object>>() {
- });
- final DataSchema
- dataSchema =
- new DataSchema(dataSourceName,
- inputParser,
- dimensionsAndAggregates.rhs,
- granularitySpec,
- null,
- DruidStorageHandlerUtils.JSON_MAPPER);
-
- IndexSpec indexSpec = DruidStorageHandlerUtils.getIndexSpec(getConf());
-
- KafkaSupervisorSpec
- spec =
- DruidKafkaUtils.createKafkaSupervisorSpec(table, kafkaTopic, kafkaServers, dataSchema, indexSpec);
-
- // Fetch existing Ingestion Spec from Druid, if any
- KafkaSupervisorSpec existingSpec = fetchKafkaIngestionSpec(table);
- String
- targetState =
- DruidStorageHandlerUtils.getTableProperty(table, DruidConstants.DRUID_KAFKA_INGESTION);
- if (targetState == null) {
- // Case when user has not specified any ingestion state in the current command
- // if there is a kafka supervisor running then keep it last known state is START otherwise STOP.
- targetState = existingSpec == null ? "STOP" : "START";
- }
-
- if ("STOP".equalsIgnoreCase(targetState)) {
- if (existingSpec != null) {
- stopKafkaIngestion(overlordAddress, dataSourceName);
- }
- } else if ("START".equalsIgnoreCase(targetState)) {
- if (existingSpec == null || !existingSpec.equals(spec)) {
- DruidKafkaUtils.updateKafkaIngestionSpec(overlordAddress, spec);
- }
- } else if ("RESET".equalsIgnoreCase(targetState)) {
- // Case when there are changes in multiple table properties.
- if (existingSpec != null && !existingSpec.equals(spec)) {
- DruidKafkaUtils.updateKafkaIngestionSpec(overlordAddress, spec);
- }
- resetKafkaIngestion(overlordAddress, dataSourceName);
- } else {
- throw new IllegalArgumentException(String.format(
- "Invalid value for property [%s], Valid values are [START, STOP, RESET]",
- DruidConstants.DRUID_KAFKA_INGESTION));
- }
- // We do not want to keep state in two separate places so remove from hive table properties.
- table.getParameters().remove(DruidConstants.DRUID_KAFKA_INGESTION);
- }
-
- private void resetKafkaIngestion(String overlordAddress, String dataSourceName) {
- try {
- StringFullResponseHolder
- response =
- RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(),
- new Request(HttpMethod.POST, new URL(
- String.format("http://%s/druid/indexer/v1/supervisor/%s/reset", overlordAddress, dataSourceName))),
- new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException,
- getMaxRetryCount());
- if (response.getStatus().equals(HttpResponseStatus.OK)) {
- CONSOLE.printInfo("Druid Kafka Ingestion Reset successful.");
- } else {
- throw new IOException(String.format("Unable to reset Kafka Ingestion Druid status [%d] full response [%s]",
- response.getStatus().getCode(),
- response.getContent()));
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
- private void stopKafkaIngestion(String overlordAddress, String dataSourceName) {
- try {
- StringFullResponseHolder
- response =
- RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(),
- new Request(HttpMethod.POST, new URL(
- String.format("http://%s/druid/indexer/v1/supervisor/%s/shutdown", overlordAddress, dataSourceName))),
- new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException,
- getMaxRetryCount());
- if (response.getStatus().equals(HttpResponseStatus.OK)) {
- CONSOLE.printInfo("Druid Kafka Ingestion shutdown successful.");
- } else {
- throw new IOException(String.format("Unable to stop Kafka Ingestion Druid status [%d] full response [%s]",
- response.getStatus().getCode(),
- response.getContent()));
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- }
-
- private KafkaSupervisorSpec fetchKafkaIngestionSpec(Table table) {
- // Stop Kafka Ingestion first
- final String
- overlordAddress =
- Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS),
- "Druid Overlord Address is null");
- String
- dataSourceName =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE),
- "Druid Datasource name is null");
- try {
- StringFullResponseHolder
- response =
- RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(),
- new Request(HttpMethod.GET,
- new URL(String.format("http://%s/druid/indexer/v1/supervisor/%s", overlordAddress, dataSourceName))),
- new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException,
- getMaxRetryCount());
- if (response.getStatus().equals(HttpResponseStatus.OK)) {
- return JSON_MAPPER.readValue(response.getContent(), KafkaSupervisorSpec.class);
- // Druid Returns 400 Bad Request when not found.
- } else if (response.getStatus().equals(HttpResponseStatus.NOT_FOUND) || response.getStatus()
- .equals(HttpResponseStatus.BAD_REQUEST)) {
- LOG.debug("No Kafka Supervisor found for datasource[%s]", dataSourceName);
- return null;
- } else {
- throw new IOException(String.format(
- "Unable to fetch Kafka Ingestion Spec from Druid status [%d] full response [%s]",
- response.getStatus().getCode(),
- response.getContent()));
- }
- } catch (Exception e) {
- throw new RuntimeException("Exception while fetching kafka ingestion spec from druid", e);
- }
- }
-
- /**
- * Fetches kafka supervisor status report from druid overlord. This method will return null if can not fetch report
- *
- * @param table object.
- * @return kafka supervisor report or null when druid overlord is unreachable.
- */
- @Nullable private KafkaSupervisorReport fetchKafkaSupervisorReport(Table table) {
- final String
- overlordAddress =
- Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS),
- "Druid Overlord Address is null");
- final String
- dataSourceName =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE),
- "Druid Datasource name is null");
- try {
- StringFullResponseHolder
- response =
- RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(),
- new Request(HttpMethod.GET, new URL(
- String.format("http://%s/druid/indexer/v1/supervisor/%s/status", overlordAddress, dataSourceName))),
- new StringFullResponseHandler(Charset.forName("UTF-8"))), input -> input instanceof IOException,
- getMaxRetryCount());
- if (response.getStatus().equals(HttpResponseStatus.OK)) {
- return DruidStorageHandlerUtils.JSON_MAPPER.readValue(response.getContent(), KafkaSupervisorReport.class);
- // Druid Returns 400 Bad Request when not found.
- } else if (response.getStatus().equals(HttpResponseStatus.NOT_FOUND) || response.getStatus()
- .equals(HttpResponseStatus.BAD_REQUEST)) {
- LOG.info("No Kafka Supervisor found for datasource[%s]", dataSourceName);
- return null;
- } else {
- LOG.error("Unable to fetch Kafka Supervisor status [%d] full response [%s]",
- response.getStatus().getCode(),
- response.getContent());
- return null;
- }
- } catch (Exception e) {
- LOG.error("Exception while fetching kafka ingestion spec from druid", e);
- return null;
- }
- }
-
- /**
- * Creates metadata moves then commit the Segment's metadata to Druid metadata store in one TxN.
- *
- * @param table Hive table
- * @param overwrite true if it is an insert overwrite table.
- */
- private List<DataSegment> loadAndCommitDruidSegments(Table table, boolean overwrite, List<DataSegment> segmentsToLoad)
- throws IOException, CallbackFailedException {
- final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
- final String
- segmentDirectory =
- table.getParameters().get(DruidConstants.DRUID_SEGMENT_DIRECTORY) != null ?
- table.getParameters().get(DruidConstants.DRUID_SEGMENT_DIRECTORY) :
- HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY);
-
- final HdfsDataSegmentPusherConfig hdfsSegmentPusherConfig = new HdfsDataSegmentPusherConfig();
- List<DataSegment> publishedDataSegmentList;
-
- LOG.info(String.format("Moving [%s] Druid segments from staging directory [%s] to Deep storage [%s]",
- segmentsToLoad.size(),
- getStagingWorkingDir().toString(),
- segmentDirectory));
- hdfsSegmentPusherConfig.setStorageDirectory(segmentDirectory);
- DataSegmentPusher dataSegmentPusher = new HdfsDataSegmentPusher(hdfsSegmentPusherConfig, getConf(), JSON_MAPPER);
- publishedDataSegmentList =
- DruidStorageHandlerUtils.publishSegmentsAndCommit(getConnector(),
- getDruidMetadataStorageTablesConfig(),
- dataSourceName,
- segmentsToLoad,
- overwrite,
- getConf(),
- dataSegmentPusher);
- return publishedDataSegmentList;
- }
-
- /**
- * This function checks the load status of Druid segments by polling druid coordinator.
- * @param segments List of druid segments to check for
- */
- private void checkLoadStatus(List<DataSegment> segments) {
- final String
- coordinatorAddress =
- HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_COORDINATOR_DEFAULT_ADDRESS);
- int maxTries = getMaxRetryCount();
-
- LOG.debug("checking load status from coordinator {}", coordinatorAddress);
-
- String coordinatorResponse;
- try {
- coordinatorResponse =
- RetryUtils.retry(() -> DruidStorageHandlerUtils.getResponseFromCurrentLeader(getHttpClient(),
- new Request(HttpMethod.GET, new URL(String.format("http://%s/status", coordinatorAddress))),
- new StringFullResponseHandler(Charset.forName("UTF-8"))).getContent(),
- input -> input instanceof IOException, maxTries);
- } catch (Exception e) {
- CONSOLE.printInfo("Will skip waiting for data loading, coordinator unavailable");
- return;
- }
- if (Strings.isNullOrEmpty(coordinatorResponse)) {
- CONSOLE.printInfo("Will skip waiting for data loading empty response from coordinator");
- }
- CONSOLE.printInfo(String.format("Waiting for the loading of [%s] segments", segments.size()));
- long passiveWaitTimeMs = HiveConf.getLongVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_PASSIVE_WAIT_TIME);
- Set<URL> urlsOfUnloadedSegments = segments
- .stream()
- .map(dataSegment -> {
- try {
- //Need to make sure that we are using segment identifier
- return new URL(String.format("http://%s/druid/coordinator/v1/datasources/%s/segments/%s",
- coordinatorAddress,
- dataSegment.getDataSource(),
- dataSegment.getId().toString()));
- } catch (MalformedURLException e) {
- Throwables.propagate(e);
- }
- return null;
- })
- .collect(Collectors.toSet());
-
- int numRetries = 0;
- while (numRetries++ < maxTries && !urlsOfUnloadedSegments.isEmpty()) {
- urlsOfUnloadedSegments = ImmutableSet.copyOf(Sets.filter(urlsOfUnloadedSegments, input -> {
- try {
- String result = DruidStorageHandlerUtils
- .getResponseFromCurrentLeader(getHttpClient(), new Request(HttpMethod.GET, input),
- new StringFullResponseHandler(Charset.forName("UTF-8"))).getContent();
-
- LOG.debug("Checking segment [{}] response is [{}]", input, result);
- return Strings.isNullOrEmpty(result);
- } catch (InterruptedException | ExecutionException e) {
- LOG.error(String.format("Error while checking URL [%s]", input), e);
- return true;
- }
- }));
-
- try {
- if (!urlsOfUnloadedSegments.isEmpty()) {
- Thread.sleep(passiveWaitTimeMs);
- }
- } catch (InterruptedException e) {
- throw new RuntimeException(e);
- }
- }
- if (!urlsOfUnloadedSegments.isEmpty()) {
- // We are not Throwing an exception since it might be a transient issue that is blocking loading
- CONSOLE.printError(String.format("Wait time exhausted and we have [%s] out of [%s] segments not loaded yet",
- urlsOfUnloadedSegments.size(),
- segments.size()));
- }
- }
-
- @VisibleForTesting void deleteSegment(DataSegment segment) throws SegmentLoadingException {
-
- final Path path = DruidStorageHandlerUtils.getPath(segment);
- LOG.info("removing segment {}, located at path {}", segment.getId().toString(), path);
-
- try {
- if (path.getName().endsWith(".zip")) {
-
- final FileSystem fs = path.getFileSystem(getConf());
-
- if (!fs.exists(path)) {
- LOG.warn("Segment Path {} does not exist. It appears to have been deleted already.", path);
- return;
- }
-
- // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip
- Path partitionNumDir = path.getParent();
- if (!fs.delete(partitionNumDir, true)) {
- throw new SegmentLoadingException("Unable to kill segment, failed to delete dir [%s]",
- partitionNumDir.toString());
- }
-
- //try to delete other directories if possible
- Path versionDir = partitionNumDir.getParent();
- if (safeNonRecursiveDelete(fs, versionDir)) {
- Path intervalDir = versionDir.getParent();
- if (safeNonRecursiveDelete(fs, intervalDir)) {
- Path dataSourceDir = intervalDir.getParent();
- safeNonRecursiveDelete(fs, dataSourceDir);
- }
- }
- } else {
- throw new SegmentLoadingException("Unknown file type[%s]", path);
- }
- } catch (IOException e) {
- throw new SegmentLoadingException(e, "Unable to kill segment");
- }
- }
-
- private static boolean safeNonRecursiveDelete(FileSystem fs, Path path) {
- try {
- return fs.delete(path, false);
- } catch (Exception ex) {
- return false;
- }
- }
-
- @Override public void preDropTable(Table table) {
- // Nothing to do
- }
-
- @Override public void rollbackDropTable(Table table) {
- // Nothing to do
- }
-
- @Override public void commitDropTable(Table table, boolean deleteData) {
- if (DruidKafkaUtils.isKafkaStreamingTable(table)) {
- // Stop Kafka Ingestion first
- final String
- overlordAddress =
- Preconditions.checkNotNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_OVERLORD_DEFAULT_ADDRESS),
- "Druid Overlord Address is null");
- String
- dataSourceName =
- Preconditions.checkNotNull(DruidStorageHandlerUtils.getTableProperty(table, Constants.DRUID_DATA_SOURCE),
- "Druid Datasource name is null");
- stopKafkaIngestion(overlordAddress, dataSourceName);
- }
-
- String
- dataSourceName =
- Preconditions.checkNotNull(table.getParameters().get(Constants.DRUID_DATA_SOURCE), "DataSource name is null !");
- // Move MetaStoreUtils.isExternalTablePurge(table) calls to a common place for all StorageHandlers
- // deleteData flag passed down to StorageHandler should be true only if
- // MetaStoreUtils.isExternalTablePurge(table) returns true.
- if (deleteData && MetaStoreUtils.isExternalTablePurge(table)) {
- LOG.info("Dropping with purge all the data for data source {}", dataSourceName);
- List<DataSegment>
- dataSegmentList =
- DruidStorageHandlerUtils.getDataSegmentList(getConnector(),
- getDruidMetadataStorageTablesConfig(),
- dataSourceName);
- if (dataSegmentList.isEmpty()) {
- LOG.info("Nothing to delete for data source {}", dataSourceName);
- return;
- }
- for (DataSegment dataSegment : dataSegmentList) {
- try {
- deleteSegment(dataSegment);
- } catch (SegmentLoadingException e) {
- LOG.error(String.format("Error while deleting segment [%s]", dataSegment.getId().toString()), e);
- }
- }
- }
- if (DruidStorageHandlerUtils.disableDataSource(getConnector(),
- getDruidMetadataStorageTablesConfig(),
- dataSourceName)) {
- LOG.info("Successfully dropped druid data source {}", dataSourceName);
- }
- }
-
- @Override public void commitInsertTable(Table table, boolean overwrite) throws MetaException {
- LOG.debug("commit insert into table {} overwrite {}", table.getTableName(), overwrite);
- try {
- // Check if there segments to load
- final Path segmentDescriptorDir = getSegmentDescriptorDir();
- final List<DataSegment> segmentsToLoad = fetchSegmentsMetadata(segmentDescriptorDir);
- final String dataSourceName = table.getParameters().get(Constants.DRUID_DATA_SOURCE);
- //No segments to load still need to honer overwrite
- if (segmentsToLoad.isEmpty() && overwrite) {
- //disable datasource
- //Case it is an insert overwrite we have to disable the existing Druid DataSource
- DruidStorageHandlerUtils.disableDataSource(getConnector(),
- getDruidMetadataStorageTablesConfig(),
- dataSourceName);
- } else if (!segmentsToLoad.isEmpty()) {
- // at this point we have Druid segments from reducers but we need to atomically
- // rename and commit to metadata
- // Moving Druid segments and committing to druid metadata as one transaction.
- checkLoadStatus(loadAndCommitDruidSegments(table, overwrite, segmentsToLoad));
- }
- } catch (IOException e) {
- throw new MetaException(e.getMessage());
- } catch (CallbackFailedException c) {
- LOG.error("Error while committing transaction to druid metadata storage", c);
- throw new MetaException(c.getCause().getMessage());
- } finally {
- cleanWorkingDir();
- }
- }
-
- private List<DataSegment> fetchSegmentsMetadata(Path segmentDescriptorDir) throws IOException {
- if (!segmentDescriptorDir.getFileSystem(getConf()).exists(segmentDescriptorDir)) {
- LOG.info("Directory {} does not exist, ignore this if it is create statement or inserts of 0 rows,"
- + " no Druid segments to move, cleaning working directory {}",
- segmentDescriptorDir.toString(),
- getStagingWorkingDir().toString());
- return Collections.emptyList();
- }
- return DruidStorageHandlerUtils.getCreatedSegments(segmentDescriptorDir, getConf());
- }
-
- @Override public void preInsertTable(Table table, boolean overwrite) {
-
- }
-
- @Override public void rollbackInsertTable(Table table, boolean overwrite) {
- // do nothing
- }
-
- @Override public void configureOutputJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) {
- jobProperties.put(Constants.DRUID_DATA_SOURCE, tableDesc.getTableName());
- jobProperties.put(DruidConstants.DRUID_SEGMENT_VERSION, new DateTime().toString());
- jobProperties.put(DruidConstants.DRUID_JOB_WORKING_DIRECTORY, getStagingWorkingDir().toString());
- // DruidOutputFormat will write segments in an intermediate directory
- jobProperties.put(DruidConstants.DRUID_SEGMENT_INTERMEDIATE_DIRECTORY,
- getIntermediateSegmentDir().toString());
- }
-
- @Override public void configureTableJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) {
-
- }
-
- @Override public void configureJobConf(TableDesc tableDesc, JobConf jobConf) {
- final boolean kerberosEnabled = HiveConf.getBoolVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_KERBEROS_ENABLE);
- if (kerberosEnabled && UserGroupInformation.isSecurityEnabled()) {
- // AM can not do Kerberos Auth so will do the input split generation in the HS2
- LOG.debug("Setting {} to {} to enable split generation on HS2",
- HiveConf.ConfVars.HIVE_AM_SPLIT_GENERATION.toString(),
- Boolean.FALSE.toString());
- jobConf.set(HiveConf.ConfVars.HIVE_AM_SPLIT_GENERATION.toString(), Boolean.FALSE.toString());
- }
- try {
- Utilities.addDependencyJars(jobConf, DruidRecordWriter.class);
- } catch (IOException e) {
- Throwables.propagate(e);
- }
- }
-
- @Override public void setConf(Configuration conf) {
- this.conf = conf;
- }
-
- @Override public Configuration getConf() {
- return conf;
- }
-
- @Override public LockType getLockType(WriteEntity writeEntity) {
- if (writeEntity.getWriteType().equals(WriteEntity.WriteType.INSERT)) {
- return LockType.SHARED_READ;
- }
- return LockType.SHARED_WRITE;
- }
-
- @Override public String toString() {
- return Constants.DRUID_HIVE_STORAGE_HANDLER_ID;
- }
-
- private String getUniqueId() {
- if (uniqueId == null) {
- uniqueId =
- Preconditions.checkNotNull(Strings.emptyToNull(HiveConf.getVar(getConf(), HiveConf.ConfVars.HIVE_QUERY_ID)),
- "Hive query id is null");
- }
- return uniqueId;
- }
-
- private Path getStagingWorkingDir() {
- return new Path(getRootWorkingDir(), makeStagingName());
- }
-
- private MetadataStorageTablesConfig getDruidMetadataStorageTablesConfig() {
- if (druidMetadataStorageTablesConfig != null) {
- return druidMetadataStorageTablesConfig;
- }
- final String base = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_BASE);
- druidMetadataStorageTablesConfig = MetadataStorageTablesConfig.fromBase(base);
- return druidMetadataStorageTablesConfig;
- }
-
- private SQLMetadataConnector getConnector() {
- return Suppliers.memoize(this::buildConnector).get();
- }
-
- private static String getPassword(Configuration conf, HiveConf.ConfVars var) {
- try {
- final char[] password = conf.getPassword(var.varname);
- return password == null ? null : String.valueOf(password);
- } catch (IOException e) {
- // Reading password from configuration for backwards compatibility.
- LOG.warn("Unable to retrieve password from credential providers. Trying to read it from config..", e);
- return conf.get(var.varname);
- }
- }
-
- private SQLMetadataConnector buildConnector() {
-
- if (connector != null) {
- return connector;
- }
-
- final String dbType = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_TYPE);
- final String username = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_USERNAME);
- final String password = getPassword(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_PASSWORD);
- final String uri = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_METADATA_DB_URI);
- LOG.debug("Supplying SQL Connector with DB type {}, URI {}, User {}", dbType, uri, username);
- @SuppressWarnings("Guava") final Supplier<MetadataStorageConnectorConfig>
- storageConnectorConfigSupplier =
- Suppliers.ofInstance(new MetadataStorageConnectorConfig() {
- @Override public String getConnectURI() {
- return uri;
- }
-
- @Override public String getUser() {
- return Strings.emptyToNull(username);
- }
-
- @Override public String getPassword() {
- return Strings.emptyToNull(password);
- }
- });
- switch (dbType) {
- case "mysql":
- connector =
- new MySQLConnector(storageConnectorConfigSupplier,
- Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()),
- new MySQLConnectorConfig());
- break;
- case "postgresql":
- connector =
- new PostgreSQLConnector(storageConnectorConfigSupplier,
- Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()), new PostgreSQLConnectorConfig(),
- new PostgreSQLTablesConfig());
-
- break;
- case "derby":
- connector =
- new DerbyConnector(new DerbyMetadataStorage(storageConnectorConfigSupplier.get()),
- storageConnectorConfigSupplier,
- Suppliers.ofInstance(getDruidMetadataStorageTablesConfig()));
- break;
- default:
- throw new IllegalStateException(String.format("Unknown metadata storage type [%s]", dbType));
- }
- return connector;
- }
-
- @VisibleForTesting String makeStagingName() {
- return ".staging-".concat(getUniqueId().replace(":", ""));
- }
-
- private Path getSegmentDescriptorDir() {
- return new Path(getStagingWorkingDir(), SEGMENTS_DESCRIPTOR_DIR_NAME);
- }
-
- private Path getIntermediateSegmentDir() {
- return new Path(getStagingWorkingDir(), INTERMEDIATE_SEGMENT_DIR_NAME);
- }
-
- private void cleanWorkingDir() {
- final FileSystem fileSystem;
- try {
- fileSystem = getStagingWorkingDir().getFileSystem(getConf());
- fileSystem.delete(getStagingWorkingDir(), true);
- } catch (IOException e) {
- LOG.error("Got Exception while cleaning working directory", e);
- }
- }
-
- private String getRootWorkingDir() {
- if (Strings.isNullOrEmpty(rootWorkingDir)) {
- rootWorkingDir = HiveConf.getVar(getConf(), HiveConf.ConfVars.DRUID_WORKING_DIR);
- }
- return rootWorkingDir;
- }
-
- private static HttpClient makeHttpClient(Lifecycle lifecycle) {
- final int
- numConnection =
- HiveConf.getIntVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_NUM_HTTP_CONNECTION);
- final Period
- readTimeout =
- new Period(HiveConf.getVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_HTTP_READ_TIMEOUT));
- LOG.info("Creating Druid HTTP client with {} max parallel connections and {}ms read timeout",
- numConnection,
- readTimeout.toStandardDuration().getMillis());
-
- final HttpClient
- httpClient =
- HttpClientInit.createClient(HttpClientConfig.builder()
- .withNumConnections(numConnection)
- .withReadTimeout(new Period(readTimeout).toStandardDuration())
- .build(), lifecycle);
- final boolean kerberosEnabled =
- HiveConf.getBoolVar(SessionState.getSessionConf(), HiveConf.ConfVars.HIVE_DRUID_KERBEROS_ENABLE);
- if (kerberosEnabled && UserGroupInformation.isSecurityEnabled()) {
- LOG.info("building Kerberos Http Client");
- return new KerberosHttpClient(httpClient);
- }
- return httpClient;
- }
-
- public static HttpClient getHttpClient() {
- return HTTP_CLIENT;
- }
-
- @Override public void preAlterTable(Table table, EnvironmentContext context) throws MetaException {
- String alterOpType = context == null ? null : context.getProperties().get(ALTER_TABLE_OPERATION_TYPE);
- // alterOpType is null in case of stats update
- if (alterOpType != null && !ALLOWED_ALTER_TYPES.contains(alterOpType)) {
- throw new MetaException("ALTER TABLE can not be used for " + alterOpType + " to a non-native table ");
- }
- if (DruidKafkaUtils.isKafkaStreamingTable(table)) {
- updateKafkaIngestion(table);
- }
- }
-
- private int getMaxRetryCount() {
- return HiveConf.getIntVar(getConf(), HiveConf.ConfVars.HIVE_DRUID_MAX_TRIES);
- }
-
- @Override public StorageHandlerInfo getStorageHandlerInfo(Table table) throws MetaException {
- if (DruidKafkaUtils.isKafkaStreamingTable(table)) {
- KafkaSupervisorReport kafkaSupervisorReport = fetchKafkaSupervisorReport(table);
- if (kafkaSupervisorReport == null) {
- return DruidStorageHandlerInfo.UNREACHABLE;
- }
- return new DruidStorageHandlerInfo(kafkaSupervisorReport);
- } else {
- // Currently we do not expose any runtime info for non-streaming tables.
- // In future extend this add more information regarding table status.
- // e.g. Total size of segments in druid, load status of table on historical nodes etc.
- return null;
- }
- }
-
- @Override public Map<String, String> getOperatorDescProperties(OperatorDesc operatorDesc,
- Map<String, String> initialProps) {
- if (operatorDesc instanceof TableScanDesc) {
- TableScanDesc tableScanDesc = (TableScanDesc) operatorDesc;
- ExprNodeGenericFuncDesc filterExpr = tableScanDesc.getFilterExpr();
- String druidQuery = initialProps.get(Constants.DRUID_QUERY_JSON);
-
- if (filterExpr != null && druidQuery != null) {
- try {
- Query query = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, BaseQuery.class);
- Query queryWithDynamicFilters = DruidStorageHandlerUtils.addDynamicFilters(query, filterExpr, conf, false);
- Map<String, String> props = Maps.newHashMap(initialProps);
- props.put(Constants.DRUID_QUERY_JSON,
- DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(queryWithDynamicFilters));
- return props;
- } catch (IOException e) {
- LOG.error("Exception while deserializing druid query. Explain plan may not have final druid query", e);
- }
- }
- }
- // Case when we do not have any additional info to add.
- return initialProps;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java
deleted file mode 100644
index e4ab53a6c84..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerInfo.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid;
-
-import org.apache.hadoop.hive.druid.json.KafkaSupervisorReport;
-import org.apache.hadoop.hive.ql.metadata.StorageHandlerInfo;
-
-/**
- * DruidStorageHandlerInfo provides a runtime information for DruidStorageHandler.
- */
-public class DruidStorageHandlerInfo implements StorageHandlerInfo {
-
- static final StorageHandlerInfo
- UNREACHABLE =
- (StorageHandlerInfo) () -> "Druid Overlord is Unreachable, Runtime Status : unknown";
-
- private final KafkaSupervisorReport kafkaSupervisorReport;
-
- DruidStorageHandlerInfo(KafkaSupervisorReport kafkaSupervisorReport) {
- this.kafkaSupervisorReport = kafkaSupervisorReport;
- }
-
- @Override public String formatAsText() {
- StringBuilder sb = new StringBuilder();
- sb.append("Druid Storage Handler Runtime Status for ")
- .append(kafkaSupervisorReport.getId())
- .append("\n")
- .append("kafkaPartitions=")
- .append(kafkaSupervisorReport.getPayload().getPartitions())
- .append("\n")
- .append("activeTasks=")
- .append(kafkaSupervisorReport.getPayload().getActiveTasks())
- .append("\n")
- .append("publishingTasks=")
- .append(kafkaSupervisorReport.getPayload().getPublishingTasks());
-
- if (kafkaSupervisorReport.getPayload().getLatestOffsets() != null) {
- sb.append("\n").append("latestOffsets=").append(kafkaSupervisorReport.getPayload().getLatestOffsets());
- }
- if (kafkaSupervisorReport.getPayload().getMinimumLag() != null) {
- sb.append("\n").append("minimumLag=").append(kafkaSupervisorReport.getPayload().getMinimumLag());
- }
- if (kafkaSupervisorReport.getPayload().getAggregateLag() != null) {
- sb.append("\n").append("aggregateLag=").append(kafkaSupervisorReport.getPayload().getAggregateLag());
- }
- if (kafkaSupervisorReport.getPayload().getOffsetsLastUpdated() != null) {
- sb.append("\n").append("lastUpdateTime=").append(kafkaSupervisorReport.getPayload().getOffsetsLastUpdated());
- }
- return sb.toString();
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
deleted file mode 100644
index 7e49b0b7c96..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/DruidStorageHandlerUtils.java
+++ /dev/null
@@ -1,1113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.InjectableValues;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.jsontype.NamedType;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Interner;
-import com.google.common.collect.Interners;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Ordering;
-import org.apache.calcite.adapter.druid.DruidQuery;
-import org.apache.calcite.sql.validate.SqlValidatorUtil;
-import org.apache.druid.common.config.NullHandling;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.StringDimensionSchema;
-import org.apache.druid.guice.BloomFilterSerializersModule;
-import org.apache.druid.jackson.DefaultObjectMapper;
-import org.apache.druid.java.util.common.JodaUtils;
-import org.apache.druid.java.util.common.MapUtils;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.java.util.emitter.EmittingLogger;
-import org.apache.druid.java.util.emitter.core.NoopEmitter;
-import org.apache.druid.java.util.emitter.service.ServiceEmitter;
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHandler;
-import org.apache.druid.java.util.http.client.response.StringFullResponseHolder;
-import org.apache.druid.math.expr.ExprMacroTable;
-import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.SQLMetadataConnector;
-import org.apache.druid.metadata.storage.mysql.MySQLConnector;
-import org.apache.druid.query.DruidProcessingConfig;
-import org.apache.druid.query.Druids;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory;
-import org.apache.druid.query.aggregation.FloatSumAggregatorFactory;
-import org.apache.druid.query.aggregation.LongSumAggregatorFactory;
-import org.apache.druid.query.expression.LikeExprMacro;
-import org.apache.druid.query.expression.RegexpExtractExprMacro;
-import org.apache.druid.query.expression.TimestampCeilExprMacro;
-import org.apache.druid.query.expression.TimestampExtractExprMacro;
-import org.apache.druid.query.expression.TimestampFloorExprMacro;
-import org.apache.druid.query.expression.TimestampFormatExprMacro;
-import org.apache.druid.query.expression.TimestampParseExprMacro;
-import org.apache.druid.query.expression.TimestampShiftExprMacro;
-import org.apache.druid.query.expression.TrimExprMacro;
-import org.apache.druid.query.filter.AndDimFilter;
-import org.apache.druid.query.filter.BloomDimFilter;
-import org.apache.druid.query.filter.BloomKFilter;
-import org.apache.druid.query.filter.BloomKFilterHolder;
-import org.apache.druid.query.filter.BoundDimFilter;
-import org.apache.druid.query.filter.DimFilter;
-import org.apache.druid.query.filter.OrDimFilter;
-import org.apache.druid.query.groupby.GroupByQuery;
-import org.apache.druid.query.ordering.StringComparator;
-import org.apache.druid.query.ordering.StringComparators;
-import org.apache.druid.query.scan.ScanQuery;
-import org.apache.druid.query.spec.MultipleIntervalSegmentSpec;
-import org.apache.druid.query.timeseries.TimeseriesQuery;
-import org.apache.druid.query.topn.TopNQuery;
-import org.apache.druid.query.topn.TopNQueryBuilder;
-import org.apache.druid.segment.IndexIO;
-import org.apache.druid.segment.IndexMergerV9;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.VirtualColumn;
-import org.apache.druid.segment.VirtualColumns;
-import org.apache.druid.segment.column.ValueType;
-import org.apache.druid.segment.data.BitmapSerdeFactory;
-import org.apache.druid.segment.data.ConciseBitmapSerdeFactory;
-import org.apache.druid.segment.data.RoaringBitmapSerdeFactory;
-import org.apache.druid.segment.indexing.granularity.GranularitySpec;
-import org.apache.druid.segment.indexing.granularity.UniformGranularitySpec;
-import org.apache.druid.segment.loading.DataSegmentPusher;
-import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
-import org.apache.druid.segment.virtual.ExpressionVirtualColumn;
-import org.apache.druid.segment.writeout.TmpFileSegmentWriteOutMediumFactory;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.TimelineObjectHolder;
-import org.apache.druid.timeline.VersionedIntervalTimeline;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.apache.druid.timeline.partition.NoneShardSpec;
-import org.apache.druid.timeline.partition.NumberedShardSpec;
-import org.apache.druid.timeline.partition.PartitionChunk;
-import org.apache.druid.timeline.partition.ShardSpec;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.druid.json.AvroParseSpec;
-import org.apache.hadoop.hive.druid.json.AvroStreamInputRowParser;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.apache.hadoop.hive.ql.exec.ExprNodeDynamicValueEvaluator;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
-import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.udf.UDFToDouble;
-import org.apache.hadoop.hive.ql.udf.UDFToFloat;
-import org.apache.hadoop.hive.ql.udf.UDFToLong;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBetween;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBridge;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInBloomFilter;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToString;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.io.retry.RetryProxy;
-import org.jboss.netty.handler.codec.http.HttpHeaders;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
-import org.joda.time.Period;
-import org.joda.time.chrono.ISOChronology;
-import org.skife.jdbi.v2.Folder3;
-import org.skife.jdbi.v2.Handle;
-import org.skife.jdbi.v2.PreparedBatch;
-import org.skife.jdbi.v2.Query;
-import org.skife.jdbi.v2.ResultIterator;
-import org.skife.jdbi.v2.exceptions.CallbackFailedException;
-import org.skife.jdbi.v2.tweak.HandleCallback;
-import org.skife.jdbi.v2.util.ByteArrayMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.annotation.Nullable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.InetAddress;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Properties;
-import java.util.Set;
-import java.util.TimeZone;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-/**
- * Utils class for Druid storage handler.
- */
-public final class DruidStorageHandlerUtils {
- private DruidStorageHandlerUtils() {
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidStorageHandlerUtils.class);
-
- private static final int NUM_RETRIES = 8;
- private static final int SECONDS_BETWEEN_RETRIES = 2;
- private static final int DEFAULT_FS_BUFFER_SIZE = 1 << 18; // 256KB
- private static final int DEFAULT_STREAMING_RESULT_SIZE = 100;
- private static final String SMILE_CONTENT_TYPE = "application/x-jackson-smile";
-
- static final String INDEX_ZIP = "index.zip";
- private static final String DESCRIPTOR_JSON = "descriptor.json";
- private static final Interval
- DEFAULT_INTERVAL =
- new Interval(new DateTime("1900-01-01", ISOChronology.getInstanceUTC()),
- new DateTime("3000-01-01", ISOChronology.getInstanceUTC())).withChronology(ISOChronology.getInstanceUTC());
-
- /**
- * Mapper to use to serialize/deserialize Druid objects (JSON).
- */
- public static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper();
-
- /**
- * Mapper to use to serialize/deserialize Druid objects (SMILE).
- */
- public static final ObjectMapper SMILE_MAPPER = new DefaultObjectMapper(new SmileFactory());
- private static final int DEFAULT_MAX_TRIES = 10;
-
- static {
- // This is needed to initliaze NullHandling for druid without guice.
- NullHandling.initializeForTests();
- // This is needed for serde of PagingSpec as it uses JacksonInject for injecting SelectQueryConfig
- InjectableValues.Std injectableValues = new InjectableValues.Std()
- // Expressions macro table used when we deserialize the query from calcite plan
- .addValue(ExprMacroTable.class, new ExprMacroTable(ImmutableList
- .of(new LikeExprMacro(), new RegexpExtractExprMacro(), new TimestampCeilExprMacro(),
- new TimestampExtractExprMacro(), new TimestampFormatExprMacro(), new TimestampParseExprMacro(),
- new TimestampShiftExprMacro(), new TimestampFloorExprMacro(), new TrimExprMacro.BothTrimExprMacro(),
- new TrimExprMacro.LeftTrimExprMacro(), new TrimExprMacro.RightTrimExprMacro())))
- .addValue(ObjectMapper.class, JSON_MAPPER)
- .addValue(DataSegment.PruneSpecsHolder.class, DataSegment.PruneSpecsHolder.DEFAULT);
-
- JSON_MAPPER.setInjectableValues(injectableValues);
- SMILE_MAPPER.setInjectableValues(injectableValues);
- // Register the shard sub type to be used by the mapper
- JSON_MAPPER.registerSubtypes(new NamedType(LinearShardSpec.class, "linear"));
- JSON_MAPPER.registerSubtypes(new NamedType(NumberedShardSpec.class, "numbered"));
- JSON_MAPPER.registerSubtypes(new NamedType(AvroParseSpec.class, "avro"));
- SMILE_MAPPER.registerSubtypes(new NamedType(AvroParseSpec.class, "avro"));
- JSON_MAPPER.registerSubtypes(new NamedType(AvroStreamInputRowParser.class, "avro_stream"));
- SMILE_MAPPER.registerSubtypes(new NamedType(AvroStreamInputRowParser.class, "avro_stream"));
- // Register Bloom Filter Serializers
- BloomFilterSerializersModule bloomFilterSerializersModule = new BloomFilterSerializersModule();
- JSON_MAPPER.registerModule(bloomFilterSerializersModule);
- SMILE_MAPPER.registerModule(bloomFilterSerializersModule);
-
- // set the timezone of the object mapper
- // THIS IS NOT WORKING workaround is to set it as part of java opts -Duser.timezone="UTC"
- JSON_MAPPER.setTimeZone(TimeZone.getTimeZone("UTC"));
- try {
- // No operation emitter will be used by some internal druid classes.
- EmittingLogger.registerEmitter(new ServiceEmitter("druid-hive-indexer",
- InetAddress.getLocalHost().getHostName(),
- new NoopEmitter()));
- } catch (UnknownHostException e) {
- throw Throwables.propagate(e);
- }
- }
-
- /**
- * Used by druid to perform IO on indexes.
- */
- public static final IndexIO INDEX_IO = new IndexIO(JSON_MAPPER, new DruidProcessingConfig() {
- @Override public String getFormatString() {
- return "%s-%s";
- }
- });
-
- /**
- * Used by druid to merge indexes.
- */
- public static final IndexMergerV9
- INDEX_MERGER_V9 =
- new IndexMergerV9(JSON_MAPPER, DruidStorageHandlerUtils.INDEX_IO, TmpFileSegmentWriteOutMediumFactory.instance());
-
- /**
- * Generic Interner implementation used to read segments object from metadata storage.
- */
- private static final Interner<DataSegment> DATA_SEGMENT_INTERNER = Interners.newWeakInterner();
-
- /**
- * Method that creates a request for Druid query using SMILE format.
- *
- * @param address of the host target.
- * @param query druid query.
- * @return Request object to be submitted.
- */
- public static Request createSmileRequest(String address, org.apache.druid.query.Query query) {
- try {
- return new Request(HttpMethod.POST, new URL(String.format("%s/druid/v2/", "http://" + address))).setContent(
- SMILE_MAPPER.writeValueAsBytes(query)).setHeader(HttpHeaders.Names.CONTENT_TYPE, SMILE_CONTENT_TYPE);
- } catch (MalformedURLException e) {
- LOG.error("URL Malformed address {}", address);
- throw new RuntimeException(e);
- } catch (JsonProcessingException e) {
- LOG.error("can not Serialize the Query [{}]", query.toString());
- throw new RuntimeException(e);
- }
- }
-
- /**
- * Method that submits a request to an Http address and retrieves the result.
- * The caller is responsible for closing the stream once it finishes consuming it.
- *
- * @param client Http Client will be used to submit request.
- * @param request Http request to be submitted.
- * @return response object.
- * @throws IOException in case of request IO error.
- */
- public static InputStream submitRequest(HttpClient client, Request request) throws IOException {
- try {
- return client.go(request, new InputStreamResponseHandler()).get();
- } catch (ExecutionException | InterruptedException e) {
- throw new IOException(e.getCause());
- }
-
- }
-
- static StringFullResponseHolder getResponseFromCurrentLeader(HttpClient client, Request request,
- StringFullResponseHandler fullResponseHandler) throws ExecutionException, InterruptedException {
- StringFullResponseHolder responseHolder = client.go(request, fullResponseHandler).get();
- if (HttpResponseStatus.TEMPORARY_REDIRECT.equals(responseHolder.getStatus())) {
- String redirectUrlStr = responseHolder.getResponse().headers().get("Location");
- LOG.debug("Request[%s] received redirect response to location [%s].", request.getUrl(), redirectUrlStr);
- final URL redirectUrl;
- try {
- redirectUrl = new URL(redirectUrlStr);
- } catch (MalformedURLException ex) {
- throw new ExecutionException(String
- .format("Malformed redirect location is found in response from url[%s], new location[%s].",
- request.getUrl(),
- redirectUrlStr), ex);
- }
- responseHolder = client.go(withUrl(request, redirectUrl), fullResponseHandler).get();
- }
- return responseHolder;
- }
-
- private static Request withUrl(Request old, URL url) {
- Request req = new Request(old.getMethod(), url);
- req.addHeaderValues(old.getHeaders());
- if (old.hasContent()) {
- req.setContent(old.getContent());
- }
- return req;
- }
-
- /**
- * @param taskDir path to the directory containing the segments descriptor info
- * the descriptor path will be
- * ../workingPath/task_id/{@link DruidStorageHandler#SEGMENTS_DESCRIPTOR_DIR_NAME}/*.json
- * @param conf hadoop conf to get the file system
- * @return List of DataSegments
- * @throws IOException can be for the case we did not produce data.
- */
- public static List<DataSegment> getCreatedSegments(Path taskDir, Configuration conf) throws IOException {
- ImmutableList.Builder<DataSegment> publishedSegmentsBuilder = ImmutableList.builder();
- FileSystem fs = taskDir.getFileSystem(conf);
- FileStatus[] fss;
- fss = fs.listStatus(taskDir);
- for (FileStatus fileStatus : fss) {
- final DataSegment segment = JSON_MAPPER.readValue((InputStream) fs.open(fileStatus.getPath()), DataSegment.class);
- publishedSegmentsBuilder.add(segment);
- }
- return publishedSegmentsBuilder.build();
- }
-
- /**
- * Writes to filesystem serialized form of segment descriptor if an existing file exists it will try to replace it.
- *
- * @param outputFS filesystem.
- * @param segment DataSegment object.
- * @param descriptorPath path.
- * @throws IOException in case any IO issues occur.
- */
- public static void writeSegmentDescriptor(final FileSystem outputFS,
- final DataSegment segment,
- final Path descriptorPath) throws IOException {
- final DataPusher descriptorPusher = (DataPusher) RetryProxy.create(DataPusher.class, () -> {
- if (outputFS.exists(descriptorPath)) {
- if (!outputFS.delete(descriptorPath, false)) {
- throw new IOException(String.format("Failed to delete descriptor at [%s]", descriptorPath));
- }
- }
- try (final OutputStream descriptorOut = outputFS.create(descriptorPath, true, DEFAULT_FS_BUFFER_SIZE)) {
- JSON_MAPPER.writeValue(descriptorOut, segment);
- descriptorOut.flush();
- }
- }, RetryPolicies.exponentialBackoffRetry(NUM_RETRIES, SECONDS_BETWEEN_RETRIES, TimeUnit.SECONDS));
- descriptorPusher.push();
- }
-
- /**
- * @param connector SQL metadata connector to the metadata storage
- * @param metadataStorageTablesConfig Table config
- * @return all the active data sources in the metadata storage
- */
- static Collection<String> getAllDataSourceNames(SQLMetadataConnector connector,
- final MetadataStorageTablesConfig metadataStorageTablesConfig) {
- return connector.getDBI()
- .withHandle((HandleCallback<List<String>>) handle -> handle.createQuery(String.format(
- "SELECT DISTINCT(datasource) FROM %s WHERE used = true",
- metadataStorageTablesConfig.getSegmentsTable()))
- .fold(Lists.<String>newArrayList(),
- (druidDataSources, stringObjectMap, foldController, statementContext) -> {
- druidDataSources.add(MapUtils.getString(stringObjectMap, "datasource"));
- return druidDataSources;
- }));
- }
-
- /**
- * @param connector SQL connector to metadata
- * @param metadataStorageTablesConfig Tables configuration
- * @param dataSource Name of data source
- * @return true if the data source was successfully disabled false otherwise
- */
- static boolean disableDataSource(SQLMetadataConnector connector,
- final MetadataStorageTablesConfig metadataStorageTablesConfig,
- final String dataSource) {
- try {
- if (!getAllDataSourceNames(connector, metadataStorageTablesConfig).contains(dataSource)) {
- LOG.warn("Cannot delete data source {}, does not exist", dataSource);
- return false;
- }
-
- connector.getDBI().withHandle((HandleCallback<Void>) handle -> {
- disableDataSourceWithHandle(handle, metadataStorageTablesConfig, dataSource);
- return null;
- });
-
- } catch (Exception e) {
- LOG.error(String.format("Error removing dataSource %s", dataSource), e);
- return false;
- }
- return true;
- }
-
- /**
- * First computes the segments timeline to accommodate new segments for insert into case.
- * Then moves segments to druid deep storage with updated metadata/version.
- * ALL IS DONE IN ONE TRANSACTION
- *
- * @param connector DBI connector to commit
- * @param metadataStorageTablesConfig Druid metadata tables definitions
- * @param dataSource Druid datasource name
- * @param segments List of segments to move and commit to metadata
- * @param overwrite if it is an insert overwrite
- * @param conf Configuration
- * @param dataSegmentPusher segment pusher
- * @return List of successfully published Druid segments.
- * This list has the updated versions and metadata about segments after move and timeline sorting
- * @throws CallbackFailedException in case the connector can not add the segment to the DB.
- */
- @SuppressWarnings("unchecked") static List<DataSegment> publishSegmentsAndCommit(final SQLMetadataConnector connector,
- final MetadataStorageTablesConfig metadataStorageTablesConfig,
- final String dataSource,
- final List<DataSegment> segments,
- boolean overwrite,
- Configuration conf,
- DataSegmentPusher dataSegmentPusher) throws CallbackFailedException {
- return connector.getDBI().inTransaction((handle, transactionStatus) -> {
- // We create the timeline for the existing and new segments
- VersionedIntervalTimeline<String, DataSegment> timeline;
- if (overwrite) {
- // If we are overwriting, we disable existing sources
- disableDataSourceWithHandle(handle, metadataStorageTablesConfig, dataSource);
-
- // When overwriting, we just start with empty timeline,
- // as we are overwriting segments with new versions
- timeline = new VersionedIntervalTimeline<>(Ordering.natural());
- } else {
- // Append Mode
- if (segments.isEmpty()) {
- // If there are no new segments, we can just bail out
- return Collections.EMPTY_LIST;
- }
- // Otherwise, build a timeline of existing segments in metadata storage
- Interval
- indexedInterval =
- JodaUtils.umbrellaInterval(segments.stream().map(DataSegment::getInterval).collect(Collectors.toList()));
- LOG.info("Building timeline for umbrella Interval [{}]", indexedInterval);
- timeline = getTimelineForIntervalWithHandle(handle, dataSource, indexedInterval, metadataStorageTablesConfig);
- }
-
- final List<DataSegment> finalSegmentsToPublish = Lists.newArrayList();
- for (DataSegment segment : segments) {
- List<TimelineObjectHolder<String, DataSegment>> existingChunks = timeline.lookup(segment.getInterval());
- if (existingChunks.size() > 1) {
- // Not possible to expand since we have more than one chunk with a single segment.
- // This is the case when user wants to append a segment with coarser granularity.
- // case metadata storage has segments with granularity HOUR and segments to append have DAY granularity.
- // Druid shard specs does not support multiple partitions for same interval with different granularity.
- throw new IllegalStateException(String.format(
- "Cannot allocate new segment for dataSource[%s], interval[%s], already have [%,d] chunks. "
- + "Not possible to append new segment.",
- dataSource,
- segment.getInterval(),
- existingChunks.size()));
- }
- // Find out the segment with latest version and maximum partition number
- SegmentIdWithShardSpec max = null;
- final ShardSpec newShardSpec;
- final String newVersion;
- if (!existingChunks.isEmpty()) {
- // Some existing chunk, Find max
- TimelineObjectHolder<String, DataSegment> existingHolder = Iterables.getOnlyElement(existingChunks);
- for (PartitionChunk<DataSegment> existing : existingHolder.getObject()) {
- if (max == null || max.getShardSpec().getPartitionNum() < existing.getObject()
- .getShardSpec()
- .getPartitionNum()) {
- max = SegmentIdWithShardSpec.fromDataSegment(existing.getObject());
- }
- }
- }
-
- if (max == null) {
- // No existing shard present in the database, use the current version.
- newShardSpec = segment.getShardSpec();
- newVersion = segment.getVersion();
- } else {
- // use version of existing max segment to generate new shard spec
- newShardSpec = getNextPartitionShardSpec(max.getShardSpec());
- newVersion = max.getVersion();
- }
- DataSegment
- publishedSegment =
- publishSegmentWithShardSpec(segment,
- newShardSpec,
- newVersion,
- getPath(segment).getFileSystem(conf),
- dataSegmentPusher);
- finalSegmentsToPublish.add(publishedSegment);
- timeline.add(publishedSegment.getInterval(),
- publishedSegment.getVersion(),
- publishedSegment.getShardSpec().createChunk(publishedSegment));
-
- }
-
- // Publish new segments to metadata storage
- final PreparedBatch
- batch =
- handle.prepareBatch(String.format(
- "INSERT INTO %1$s (id, dataSource, created_date, start, \"end\", partitioned, version, used, payload) "
- + "VALUES (:id, :dataSource, :created_date, :start, :end, :partitioned, :version, :used, :payload)",
- metadataStorageTablesConfig.getSegmentsTable())
-
- );
-
- for (final DataSegment segment : finalSegmentsToPublish) {
-
- batch.add(new ImmutableMap.Builder<String, Object>().put("id", segment.getId().toString())
- .put("dataSource", segment.getDataSource())
- .put("created_date", new DateTime().toString())
- .put("start", segment.getInterval().getStart().toString())
- .put("end", segment.getInterval().getEnd().toString())
- .put("partitioned", !(segment.getShardSpec() instanceof NoneShardSpec))
- .put("version", segment.getVersion())
- .put("used", true)
- .put("payload", JSON_MAPPER.writeValueAsBytes(segment))
- .build());
-
- LOG.info("Published {}", segment.getId().toString());
- }
- batch.execute();
-
- return finalSegmentsToPublish;
- });
- }
-
- private static void disableDataSourceWithHandle(Handle handle,
- MetadataStorageTablesConfig metadataStorageTablesConfig,
- String dataSource) {
- handle.createStatement(String.format("UPDATE %s SET used=false WHERE dataSource = :dataSource",
- metadataStorageTablesConfig.getSegmentsTable())).bind("dataSource", dataSource).execute();
- }
-
- /**
- * @param connector SQL connector to metadata
- * @param metadataStorageTablesConfig Tables configuration
- * @param dataSource Name of data source
- * @return List of all data segments part of the given data source
- */
- static List<DataSegment> getDataSegmentList(final SQLMetadataConnector connector,
- final MetadataStorageTablesConfig metadataStorageTablesConfig,
- final String dataSource) {
- return connector.retryTransaction((handle, status) -> handle.createQuery(String.format(
- "SELECT payload FROM %s WHERE dataSource = :dataSource",
- metadataStorageTablesConfig.getSegmentsTable()))
- .setFetchSize(getStreamingFetchSize(connector))
- .bind("dataSource", dataSource)
- .map(ByteArrayMapper.FIRST)
- .fold(new ArrayList<>(), (Folder3<List<DataSegment>, byte[]>) (accumulator, payload, control, ctx) -> {
- try {
- final DataSegment segment = DATA_SEGMENT_INTERNER.intern(JSON_MAPPER.readValue(payload, DataSegment.class));
-
- accumulator.add(segment);
- return accumulator;
- } catch (Exception e) {
- throw new SQLException(e.toString());
- }
- }), 3, DEFAULT_MAX_TRIES);
- }
-
- /**
- * @param connector SQL DBI connector.
- * @return streaming fetch size.
- */
- private static int getStreamingFetchSize(SQLMetadataConnector connector) {
- if (connector instanceof MySQLConnector) {
- return Integer.MIN_VALUE;
- }
- return DEFAULT_STREAMING_RESULT_SIZE;
- }
-
- /**
- * @param pushedSegment the pushed data segment object
- * @param segmentsDescriptorDir actual directory path for descriptors.
- * @return a sanitize file name
- */
- public static Path makeSegmentDescriptorOutputPath(DataSegment pushedSegment, Path segmentsDescriptorDir) {
- return new Path(segmentsDescriptorDir, String.format("%s.json", pushedSegment.getId().toString().replace(":", "")));
- }
-
- public static String createScanAllQuery(String dataSourceName, List<String> columns) throws JsonProcessingException {
- final Druids.ScanQueryBuilder scanQueryBuilder = Druids.newScanQueryBuilder();
- final List<Interval> intervals = Collections.singletonList(DEFAULT_INTERVAL);
- ScanQuery
- scanQuery =
- scanQueryBuilder.dataSource(dataSourceName).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
- .intervals(new MultipleIntervalSegmentSpec(intervals))
- .columns(columns)
- .build();
- return JSON_MAPPER.writeValueAsString(scanQuery);
- }
-
- @Nullable static Boolean getBooleanProperty(Table table, String propertyName) {
- String val = getTableProperty(table, propertyName);
- if (val == null) {
- return null;
- }
- return Boolean.parseBoolean(val);
- }
-
- static boolean getBooleanProperty(Table table, String propertyName, boolean defaultVal) {
- Boolean val = getBooleanProperty(table, propertyName);
- return val == null ? defaultVal : val;
- }
-
- @Nullable static Integer getIntegerProperty(Table table, String propertyName) {
- String val = getTableProperty(table, propertyName);
- if (val == null) {
- return null;
- }
- try {
- return Integer.parseInt(val);
- } catch (NumberFormatException e) {
- throw new NumberFormatException(String.format("Exception while parsing property[%s] with Value [%s] as Integer",
- propertyName,
- val));
- }
- }
-
- static int getIntegerProperty(Table table, String propertyName, int defaultVal) {
- Integer val = getIntegerProperty(table, propertyName);
- return val == null ? defaultVal : val;
- }
-
- @Nullable static Long getLongProperty(Table table, String propertyName) {
- String val = getTableProperty(table, propertyName);
- if (val == null) {
- return null;
- }
- try {
- return Long.parseLong(val);
- } catch (NumberFormatException e) {
- throw new NumberFormatException(String.format("Exception while parsing property[%s] with Value [%s] as Long",
- propertyName,
- val));
- }
- }
-
- @Nullable static Period getPeriodProperty(Table table, String propertyName) {
- String val = getTableProperty(table, propertyName);
- if (val == null) {
- return null;
- }
- try {
- return Period.parse(val);
- } catch (IllegalArgumentException e) {
- throw new IllegalArgumentException(String.format("Exception while parsing property[%s] with Value [%s] as Period",
- propertyName,
- val));
- }
- }
-
- @Nullable public static List<String> getListProperty(Table table, String propertyName) {
- List<String> rv = new ArrayList<>();
- String values = getTableProperty(table, propertyName);
- if (values == null) {
- return null;
- }
- String[] vals = values.trim().split(",");
- for (String val : vals) {
- if (org.apache.commons.lang3.StringUtils.isNotBlank(val)) {
- rv.add(val);
- }
- }
- return rv;
- }
-
- static String getTableProperty(Table table, String propertyName) {
- return table.getParameters().get(propertyName);
- }
-
- /**
- * Simple interface for retry operations.
- */
- public interface DataPusher {
- void push() throws IOException;
- }
-
- private static VersionedIntervalTimeline<String, DataSegment> getTimelineForIntervalWithHandle(final Handle handle,
- final String dataSource,
- final Interval interval,
- final MetadataStorageTablesConfig dbTables) throws IOException {
- Query<Map<String, Object>>
- sql =
- handle.createQuery(String.format(
- "SELECT payload FROM %s WHERE used = true AND dataSource = ? AND start <= ? AND \"end\" >= ?",
- dbTables.getSegmentsTable()))
- .bind(0, dataSource)
- .bind(1, interval.getEnd().toString())
- .bind(2, interval.getStart().toString());
-
- final VersionedIntervalTimeline<String, DataSegment> timeline = new VersionedIntervalTimeline<>(Ordering.natural());
- try (ResultIterator<byte[]> dbSegments = sql.map(ByteArrayMapper.FIRST).iterator()) {
- while (dbSegments.hasNext()) {
- final byte[] payload = dbSegments.next();
- DataSegment segment = JSON_MAPPER.readValue(payload, DataSegment.class);
- timeline.add(segment.getInterval(), segment.getVersion(), segment.getShardSpec().createChunk(segment));
- }
- }
- return timeline;
- }
-
- public static DataSegmentPusher createSegmentPusherForDirectory(String segmentDirectory, Configuration configuration)
- throws IOException {
- final HdfsDataSegmentPusherConfig hdfsDataSegmentPusherConfig = new HdfsDataSegmentPusherConfig();
- hdfsDataSegmentPusherConfig.setStorageDirectory(segmentDirectory);
- return new HdfsDataSegmentPusher(hdfsDataSegmentPusherConfig, configuration, JSON_MAPPER);
- }
-
- private static DataSegment publishSegmentWithShardSpec(DataSegment segment,
- ShardSpec shardSpec,
- String version,
- FileSystem fs,
- DataSegmentPusher dataSegmentPusher) throws IOException {
- boolean retry = true;
- DataSegment.Builder dataSegmentBuilder = new DataSegment.Builder(segment).version(version);
- Path finalPath = null;
- while (retry) {
- retry = false;
- dataSegmentBuilder.shardSpec(shardSpec);
- final Path intermediatePath = getPath(segment);
-
- finalPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(dataSegmentBuilder.build(), DruidStorageHandlerUtils.INDEX_ZIP));
- // Create parent if it does not exist, recreation is not an error
- fs.mkdirs(finalPath.getParent());
-
- if (!fs.rename(intermediatePath, finalPath)) {
- if (fs.exists(finalPath)) {
- // Someone else is also trying to append
- shardSpec = getNextPartitionShardSpec(shardSpec);
- retry = true;
- } else {
- throw new IOException(String.format(
- "Failed to rename intermediate segment[%s] to final segment[%s] is not present.",
- intermediatePath,
- finalPath));
- }
- }
- }
- DataSegment dataSegment = dataSegmentBuilder.loadSpec(dataSegmentPusher.makeLoadSpec(finalPath.toUri())).build();
-
- writeSegmentDescriptor(fs, dataSegment, new Path(finalPath.getParent(), DruidStorageHandlerUtils.DESCRIPTOR_JSON));
-
- return dataSegment;
- }
-
- private static ShardSpec getNextPartitionShardSpec(ShardSpec shardSpec) {
- if (shardSpec instanceof LinearShardSpec) {
- return new LinearShardSpec(shardSpec.getPartitionNum() + 1);
- } else if (shardSpec instanceof NumberedShardSpec) {
- return new NumberedShardSpec(shardSpec.getPartitionNum(), ((NumberedShardSpec) shardSpec).getPartitions());
- } else {
- // Druid only support appending more partitions to Linear and Numbered ShardSpecs.
- throw new IllegalStateException(String.format("Cannot expand shard spec [%s]", shardSpec));
- }
- }
-
- static Path getPath(DataSegment dataSegment) {
- return new Path(String.valueOf(Objects.requireNonNull(dataSegment.getLoadSpec()).get("path")));
- }
-
- public static GranularitySpec getGranularitySpec(Configuration configuration, Properties tableProperties) {
- final String
- segmentGranularity =
- tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) != null ?
- tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY) :
- HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_INDEXING_GRANULARITY);
- final boolean
- rollup =
- tableProperties.getProperty(DruidConstants.DRUID_ROLLUP) != null ?
- Boolean.parseBoolean(tableProperties.getProperty(Constants.DRUID_SEGMENT_GRANULARITY)) :
- HiveConf.getBoolVar(configuration, HiveConf.ConfVars.HIVE_DRUID_ROLLUP);
- return new UniformGranularitySpec(Granularity.fromString(segmentGranularity),
- Granularity.fromString(tableProperties.getProperty(DruidConstants.DRUID_QUERY_GRANULARITY) == null ?
- "NONE" :
- tableProperties.getProperty(DruidConstants.DRUID_QUERY_GRANULARITY)),
- rollup,
- null);
- }
-
- public static IndexSpec getIndexSpec(Configuration jc) {
- final BitmapSerdeFactory bitmapSerdeFactory;
- if ("concise".equals(HiveConf.getVar(jc, HiveConf.ConfVars.HIVE_DRUID_BITMAP_FACTORY_TYPE))) {
- bitmapSerdeFactory = new ConciseBitmapSerdeFactory();
- } else {
- bitmapSerdeFactory = new RoaringBitmapSerdeFactory(true);
- }
- return new IndexSpec(bitmapSerdeFactory,
- IndexSpec.DEFAULT_DIMENSION_COMPRESSION,
- IndexSpec.DEFAULT_METRIC_COMPRESSION,
- IndexSpec.DEFAULT_LONG_ENCODING);
- }
-
- public static Pair<List<DimensionSchema>, AggregatorFactory[]> getDimensionsAndAggregates(List<String> columnNames,
- List<TypeInfo> columnTypes) {
- // Default, all columns that are not metrics or timestamp, are treated as dimensions
- final List<DimensionSchema> dimensions = new ArrayList<>();
- ImmutableList.Builder<AggregatorFactory> aggregatorFactoryBuilder = ImmutableList.builder();
- for (int i = 0; i < columnTypes.size(); i++) {
- final PrimitiveObjectInspector.PrimitiveCategory
- primitiveCategory =
- ((PrimitiveTypeInfo) columnTypes.get(i)).getPrimitiveCategory();
- AggregatorFactory af;
- switch (primitiveCategory) {
- case BYTE:
- case SHORT:
- case INT:
- case LONG:
- af = new LongSumAggregatorFactory(columnNames.get(i), columnNames.get(i));
- break;
- case FLOAT:
- af = new FloatSumAggregatorFactory(columnNames.get(i), columnNames.get(i));
- break;
- case DOUBLE:
- af = new DoubleSumAggregatorFactory(columnNames.get(i), columnNames.get(i));
- break;
- case DECIMAL:
- throw new UnsupportedOperationException(String.format("Druid does not support decimal column type cast column "
- + "[%s] to double", columnNames.get(i)));
- case TIMESTAMP:
- // Granularity column
- String tColumnName = columnNames.get(i);
- if (!tColumnName.equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME)
- && !tColumnName.equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- throw new IllegalArgumentException("Dimension "
- + tColumnName
- + " does not have STRING type: "
- + primitiveCategory);
- }
- continue;
- case TIMESTAMPLOCALTZ:
- // Druid timestamp column
- String tLocalTZColumnName = columnNames.get(i);
- if (!tLocalTZColumnName.equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- throw new IllegalArgumentException("Dimension "
- + tLocalTZColumnName
- + " does not have STRING type: "
- + primitiveCategory);
- }
- continue;
- default:
- // Dimension
- String dColumnName = columnNames.get(i);
- if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(primitiveCategory)
- != PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP
- && primitiveCategory != PrimitiveObjectInspector.PrimitiveCategory.BOOLEAN) {
- throw new IllegalArgumentException("Dimension "
- + dColumnName
- + " does not have STRING type: "
- + primitiveCategory);
- }
- dimensions.add(new StringDimensionSchema(dColumnName));
- continue;
- }
- aggregatorFactoryBuilder.add(af);
- }
- ImmutableList<AggregatorFactory> aggregatorFactories = aggregatorFactoryBuilder.build();
- return Pair.of(dimensions, aggregatorFactories.toArray(new AggregatorFactory[0]));
- }
-
- // Druid only supports String,Long,Float,Double selectors
- private static Set<TypeInfo> druidSupportedTypeInfos =
- ImmutableSet.<TypeInfo>of(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.charTypeInfo,
- TypeInfoFactory.varcharTypeInfo, TypeInfoFactory.byteTypeInfo, TypeInfoFactory.intTypeInfo,
- TypeInfoFactory.longTypeInfo, TypeInfoFactory.shortTypeInfo, TypeInfoFactory.doubleTypeInfo);
-
- private static Set<TypeInfo> stringTypeInfos =
- ImmutableSet.<TypeInfo>of(TypeInfoFactory.stringTypeInfo, TypeInfoFactory.charTypeInfo,
- TypeInfoFactory.varcharTypeInfo);
-
- public static org.apache.druid.query.Query addDynamicFilters(org.apache.druid.query.Query query,
- ExprNodeGenericFuncDesc filterExpr, Configuration conf, boolean resolveDynamicValues) {
- List<VirtualColumn> virtualColumns = Arrays.asList(getVirtualColumns(query).getVirtualColumns());
- org.apache.druid.query.Query rv = query;
- DimFilter joinReductionFilter = toDruidFilter(filterExpr, conf, virtualColumns, resolveDynamicValues);
- if (joinReductionFilter != null) {
- String type = query.getType();
- DimFilter filter = new AndDimFilter(joinReductionFilter, query.getFilter());
- switch (type) {
- case org.apache.druid.query.Query.TIMESERIES:
- rv = Druids.TimeseriesQueryBuilder.copy((TimeseriesQuery) query).filters(filter)
- .virtualColumns(VirtualColumns.create(virtualColumns)).build();
- break;
- case org.apache.druid.query.Query.TOPN:
- rv = new TopNQueryBuilder((TopNQuery) query).filters(filter)
- .virtualColumns(VirtualColumns.create(virtualColumns)).build();
- break;
- case org.apache.druid.query.Query.GROUP_BY:
- rv = new GroupByQuery.Builder((GroupByQuery) query).setDimFilter(filter)
- .setVirtualColumns(VirtualColumns.create(virtualColumns)).build();
- break;
- case org.apache.druid.query.Query.SCAN:
- rv = Druids.ScanQueryBuilder.copy((ScanQuery) query).filters(filter)
- .virtualColumns(VirtualColumns.create(virtualColumns)).build();
- break;
- default:
- throw new UnsupportedOperationException("Unsupported Query type " + type);
- }
- }
- return rv;
- }
-
- @Nullable private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration,
- List<VirtualColumn> virtualColumns, boolean resolveDynamicValues) {
- if (filterExpr == null) {
- return null;
- }
- Class<? extends GenericUDF> genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr);
- if (FunctionRegistry.isOpAnd(filterExpr)) {
- Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
- List<DimFilter> delegates = Lists.newArrayList();
- while (iterator.hasNext()) {
- DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
- if (filter != null) {
- delegates.add(filter);
- }
- }
- if (!delegates.isEmpty()) {
- return new AndDimFilter(delegates);
- }
- }
- if (FunctionRegistry.isOpOr(filterExpr)) {
- Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
- List<DimFilter> delegates = Lists.newArrayList();
- while (iterator.hasNext()) {
- DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
- if (filter != null) {
- delegates.add(filter);
- }
- }
- if (!delegates.isEmpty()) {
- return new OrDimFilter(delegates);
- }
- } else if (GenericUDFBetween.class == genericUDFClass) {
- List<ExprNodeDesc> child = filterExpr.getChildren();
- String col = extractColName(child.get(1), virtualColumns);
- if (col != null) {
- try {
- StringComparator comparator = stringTypeInfos
- .contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC;
- String lower = evaluate(child.get(2), configuration, resolveDynamicValues);
- String upper = evaluate(child.get(3), configuration, resolveDynamicValues);
- return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator);
-
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
- }
- } else if (GenericUDFInBloomFilter.class == genericUDFClass) {
- List<ExprNodeDesc> child = filterExpr.getChildren();
- String col = extractColName(child.get(0), virtualColumns);
- if (col != null) {
- try {
- BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues);
- return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null);
- } catch (HiveException | IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
- return null;
- }
-
- private static String evaluate(ExprNodeDesc desc, Configuration configuration, boolean resolveDynamicValue)
- throws HiveException {
- ExprNodeEvaluator exprNodeEvaluator = ExprNodeEvaluatorFactory.get(desc, configuration);
- if (exprNodeEvaluator instanceof ExprNodeDynamicValueEvaluator && !resolveDynamicValue) {
- return desc.getExprStringForExplain();
- } else {
- return exprNodeEvaluator.evaluate(null).toString();
- }
- }
-
- private static BloomKFilter evaluateBloomFilter(ExprNodeDesc desc, Configuration configuration,
- boolean resolveDynamicValue) throws HiveException, IOException {
- if (!resolveDynamicValue) {
- // return a dummy bloom filter for explain
- return new BloomKFilter(1);
- } else {
- BytesWritable bw = (BytesWritable) ExprNodeEvaluatorFactory.get(desc, configuration).evaluate(null);
- return BloomKFilter.deserialize(ByteBuffer.wrap(bw.getBytes()));
- }
- }
-
- @Nullable public static String extractColName(ExprNodeDesc expr, List<VirtualColumn> virtualColumns) {
- if (!druidSupportedTypeInfos.contains(expr.getTypeInfo())) {
- // This column type is currently not supported in druid.(e.g boolean)
- // We cannot pass the bloom filter to druid since bloom filter tests for exact object bytes.
- return null;
- }
- if (expr instanceof ExprNodeColumnDesc) {
- return ((ExprNodeColumnDesc) expr).getColumn();
- }
-
- ExprNodeGenericFuncDesc funcDesc = null;
- if (expr instanceof ExprNodeGenericFuncDesc) {
- funcDesc = (ExprNodeGenericFuncDesc) expr;
- }
- if (null == funcDesc) {
- return null;
- }
- GenericUDF udf = funcDesc.getGenericUDF();
- // bail out if its not a simple cast expression.
- if (funcDesc.getChildren().size() == 1 && funcDesc.getChildren().get(0) instanceof ExprNodeColumnDesc) {
- return null;
- }
- String columnName = ((ExprNodeColumnDesc) (funcDesc.getChildren().get(0))).getColumn();
- ValueType targetType = null;
- if (udf instanceof GenericUDFBridge) {
- Class<? extends UDF> udfClass = ((GenericUDFBridge) udf).getUdfClass();
- if (udfClass.equals(UDFToDouble.class)) {
- targetType = ValueType.DOUBLE;
- } else if (udfClass.equals(UDFToFloat.class)) {
- targetType = ValueType.FLOAT;
- } else if (udfClass.equals(UDFToLong.class)) {
- targetType = ValueType.LONG;
- }
- } else if (udf instanceof GenericUDFToString) {
- targetType = ValueType.STRING;
- }
-
- if (targetType == null) {
- return null;
- }
- String virtualColumnExpr = DruidQuery.format("CAST(%s, '%s')", columnName, targetType.toString());
- for (VirtualColumn column : virtualColumns) {
- if (column instanceof ExpressionVirtualColumn && ((ExpressionVirtualColumn) column).getExpression()
- .equals(virtualColumnExpr)) {
- // Found an existing virtual column with same expression, no need to add another virtual column
- return column.getOutputName();
- }
- }
- Set<String> usedColumnNames = virtualColumns.stream().map(col -> col.getOutputName()).collect(Collectors.toSet());
- final String name = SqlValidatorUtil.uniquify("vc", usedColumnNames, SqlValidatorUtil.EXPR_SUGGESTER);
- ExpressionVirtualColumn expressionVirtualColumn =
- new ExpressionVirtualColumn(name, virtualColumnExpr, targetType, ExprMacroTable.nil());
- virtualColumns.add(expressionVirtualColumn);
- return name;
- }
-
- public static VirtualColumns getVirtualColumns(org.apache.druid.query.Query query) {
- String type = query.getType();
- switch (type) {
- case org.apache.druid.query.Query.TIMESERIES:
- return ((TimeseriesQuery) query).getVirtualColumns();
- case org.apache.druid.query.Query.TOPN:
- return ((TopNQuery) query).getVirtualColumns();
- case org.apache.druid.query.Query.GROUP_BY:
- return ((GroupByQuery) query).getVirtualColumns();
- case org.apache.druid.query.Query.SCAN:
- return ((ScanQuery) query).getVirtualColumns();
- default:
- throw new UnsupportedOperationException("Unsupported Query type " + query);
- }
- }
-
- @Nullable private static Class<? extends GenericUDF> getGenericUDFClassFromExprDesc(ExprNodeDesc desc) {
- if (!(desc instanceof ExprNodeGenericFuncDesc)) {
- return null;
- }
- ExprNodeGenericFuncDesc genericFuncDesc = (ExprNodeGenericFuncDesc) desc;
- return genericFuncDesc.getGenericUDF().getClass();
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java
deleted file mode 100644
index 07058040662..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/DruidConstants.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.conf;
-
-/**
- * Utility class for Druid Constants.
- */
-public final class DruidConstants {
- private DruidConstants() {
- }
-
- public static final String DRUID_QUERY_FETCH = "druid.query.fetch";
-
- public static final String DRUID_ROLLUP = "druid.rollup";
-
- public static final String DRUID_QUERY_GRANULARITY = "druid.query.granularity";
-
- public static final String DRUID_SEGMENT_DIRECTORY = "druid.storage.storageDirectory";
-
- public static final String DRUID_SEGMENT_INTERMEDIATE_DIRECTORY = "druid.storage.storageDirectory.intermediate";
-
- public static final String DRUID_SEGMENT_VERSION = "druid.segment.version";
-
- public static final String DRUID_JOB_WORKING_DIRECTORY = "druid.job.workingDirectory";
-
- public static final String KAFKA_TOPIC = "kafka.topic";
-
- public static final String KAFKA_BOOTSTRAP_SERVERS = "kafka.bootstrap.servers";
-
- public static final String DRUID_KAFKA_INGESTION_PROPERTY_PREFIX = "druid.kafka.ingestion.";
-
- public static final String DRUID_KAFKA_CONSUMER_PROPERTY_PREFIX = DRUID_KAFKA_INGESTION_PROPERTY_PREFIX + "consumer.";
-
- /* Kafka Ingestion state - valid values - START/STOP/RESET */
- public static final String DRUID_KAFKA_INGESTION = "druid.kafka.ingestion";
-
- //Druid storage timestamp column name
- public static final String DEFAULT_TIMESTAMP_COLUMN = "__time";
-
- public static final String DRUID_TIMESTAMP_FORMAT = "druid.timestamp.format";
-
- // Used when the field name in ingested data via streaming ingestion does not match
- // druid default timestamp column i.e `__time`
- public static final String DRUID_TIMESTAMP_COLUMN = "druid.timestamp.column";
-
- //Druid Json timestamp column name for GroupBy results
- public static final String EVENT_TIMESTAMP_COLUMN = "timestamp";
-
- // Druid ParseSpec Type - JSON/CSV/TSV/AVRO
- public static final String DRUID_PARSE_SPEC_FORMAT = "druid.parseSpec.format";
-
- public static final String AVRO_SCHEMA_LITERAL = "avro.schema.literal";
-
- // value delimiter for druid columns
- public static final String DRUID_PARSE_SPEC_DELIMITER = "druid.parseSpec.delimiter";
-
- // list delimiter for multi-valued columns
- public static final String DRUID_PARSE_SPEC_LIST_DELIMITER = "druid.parseSpec.listDelimiter";
-
- // order of columns for delimiter and csv parse specs.
- public static final String DRUID_PARSE_SPEC_COLUMNS = "druid.parseSpec.columns";
-
- public static final String DRUID_PARSE_SPEC_SKIP_HEADER_ROWS = "druid.parseSpec.skipHeaderRows";
-
- public static final String DRUID_PARSE_SPEC_HAS_HEADER_ROWS = "druid.parseSpec.hasHeaderRows";
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java
deleted file mode 100644
index 1049acb2a94..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/conf/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package info file.
- */
-
-package org.apache.hadoop.hive.druid.conf;
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
deleted file mode 100644
index d90db9cbda9..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidOutputFormat.java
+++ /dev/null
@@ -1,178 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.io;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import org.apache.druid.data.input.impl.DimensionSchema;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.InputRowParser;
-import org.apache.druid.data.input.impl.MapInputRowParser;
-import org.apache.druid.data.input.impl.TimeAndDimsParseSpec;
-import org.apache.druid.data.input.impl.TimestampSpec;
-import org.apache.druid.java.util.common.Pair;
-import org.apache.druid.query.aggregation.AggregatorFactory;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.indexing.RealtimeTuningConfig;
-import org.apache.druid.segment.indexing.granularity.GranularitySpec;
-import org.apache.druid.segment.realtime.plumber.CustomVersioningPolicy;
-
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.druid.serde.DruidWritable;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.util.Progressable;
-
-import static org.apache.hadoop.hive.druid.DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-
-/**
- * Druid Output format class used to write data as Native Druid Segment.
- */
-public class DruidOutputFormat implements HiveOutputFormat<NullWritable, DruidWritable> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidOutputFormat.class);
-
- @Override
- public FileSinkOperator.RecordWriter getHiveRecordWriter(
- JobConf jc,
- Path finalOutPath,
- Class<? extends Writable> valueClass,
- boolean isCompressed,
- Properties tableProperties,
- Progressable progress
- ) throws IOException {
-
-
- final int targetNumShardsPerGranularity = Integer.parseUnsignedInt(
- tableProperties.getProperty(Constants.DRUID_TARGET_SHARDS_PER_GRANULARITY, "0"));
- final int maxPartitionSize = targetNumShardsPerGranularity > 0 ? -1 : HiveConf
- .getIntVar(jc, HiveConf.ConfVars.HIVE_DRUID_MAX_PARTITION_SIZE);
- // If datasource is in the table properties, it is an INSERT/INSERT OVERWRITE as the datasource
- // name was already persisted. Otherwise, it is a CT/CTAS and we need to get the name from the
- // job properties that are set by configureOutputJobProperties in the DruidStorageHandler
- final String dataSource = tableProperties.getProperty(Constants.DRUID_DATA_SOURCE) == null
- ? jc.get(Constants.DRUID_DATA_SOURCE)
- : tableProperties.getProperty(Constants.DRUID_DATA_SOURCE);
- final String segmentDirectory = jc.get(DruidConstants.DRUID_SEGMENT_INTERMEDIATE_DIRECTORY);
-
- final GranularitySpec granularitySpec = DruidStorageHandlerUtils.getGranularitySpec(jc, tableProperties);
-
- final String columnNameProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMNS);
- final String columnTypeProperty = tableProperties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
-
- if (StringUtils.isEmpty(columnNameProperty) || StringUtils.isEmpty(columnTypeProperty)) {
- throw new IllegalStateException(
- String.format("List of columns names [%s] or columns type [%s] is/are not present",
- columnNameProperty, columnTypeProperty
- ));
- }
- ArrayList<String> columnNames = Lists.newArrayList(columnNameProperty.split(","));
- if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- throw new IllegalStateException("Timestamp column (' " + DruidConstants.DEFAULT_TIMESTAMP_COLUMN +
- "') not specified in create table; list of columns is : " +
- tableProperties.getProperty(serdeConstants.LIST_COLUMNS));
- }
- ArrayList<TypeInfo> columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
-
- Pair<List<DimensionSchema>, AggregatorFactory[]> dimensionsAndAggregates = DruidStorageHandlerUtils
- .getDimensionsAndAggregates(columnNames, columnTypes);
- final InputRowParser inputRowParser = new MapInputRowParser(new TimeAndDimsParseSpec(
- new TimestampSpec(DruidConstants.DEFAULT_TIMESTAMP_COLUMN, "auto", null),
- new DimensionsSpec(dimensionsAndAggregates.lhs, Lists
- .newArrayList(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME,
- Constants.DRUID_SHARD_KEY_COL_NAME
- ), null
- )
- ));
-
- Map<String, Object>
- inputParser =
- DruidStorageHandlerUtils.JSON_MAPPER.convertValue(inputRowParser, new TypeReference<Map<String, Object>>() {
- });
-
- final DataSchema dataSchema = new DataSchema(
- Preconditions.checkNotNull(dataSource, "Data source name is null"),
- inputParser,
- dimensionsAndAggregates.rhs,
- granularitySpec,
- null,
- DruidStorageHandlerUtils.JSON_MAPPER
- );
-
- final String workingPath = jc.get(DruidConstants.DRUID_JOB_WORKING_DIRECTORY);
- final String version = jc.get(DruidConstants.DRUID_SEGMENT_VERSION);
- String basePersistDirectory = HiveConf
- .getVar(jc, HiveConf.ConfVars.HIVE_DRUID_BASE_PERSIST_DIRECTORY);
- if (Strings.isNullOrEmpty(basePersistDirectory)) {
- basePersistDirectory = System.getProperty("java.io.tmpdir");
- }
- Integer maxRowInMemory = HiveConf.getIntVar(jc, HiveConf.ConfVars.HIVE_DRUID_MAX_ROW_IN_MEMORY);
-
- IndexSpec indexSpec = DruidStorageHandlerUtils.getIndexSpec(jc);
- RealtimeTuningConfig realtimeTuningConfig = new RealtimeTuningConfig(maxRowInMemory, null, null, null,
- new File(basePersistDirectory, dataSource), new CustomVersioningPolicy(version), null, null, null, indexSpec,
- null, true, 0, 0, true, null, 0L, null, null);
-
- LOG.debug(String.format("running with Data schema [%s] ", dataSchema));
- return new DruidRecordWriter(dataSchema, realtimeTuningConfig,
- DruidStorageHandlerUtils.createSegmentPusherForDirectory(segmentDirectory, jc),
- maxPartitionSize, new Path(workingPath, SEGMENTS_DESCRIPTOR_DIR_NAME),
- finalOutPath.getFileSystem(jc)
- );
- }
-
- @Override
- public RecordWriter<NullWritable, DruidWritable> getRecordWriter(
- FileSystem ignored, JobConf job, String name, Progressable progress
- ) throws IOException {
- throw new UnsupportedOperationException("please implement me !");
- }
-
- @Override
- public void checkOutputSpecs(FileSystem ignored, JobConf job) throws IOException {
- // NOOP
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
deleted file mode 100644
index 2a2be067125..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidQueryBasedInputFormat.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.io;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.StringEscapeUtils;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.query.BaseQuery;
-import org.apache.druid.query.LocatedSegmentDescriptor;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.SegmentDescriptor;
-import org.apache.druid.query.scan.ScanQuery;
-import org.apache.druid.query.spec.MultipleSpecificSegmentSpec;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.DruidStorageHandler;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.serde.DruidGroupByQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidScanQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidTimeseriesQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidTopNQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidWritable;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedSupport;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapreduce.InputFormat;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.jboss.netty.handler.codec.http.HttpMethod;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.net.URLEncoder;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Druid query based input format.
- *
- * Given a query and the Druid broker address, it will send it, and retrieve
- * and parse the results.
- */
-public class DruidQueryBasedInputFormat extends InputFormat<NullWritable, DruidWritable>
- implements org.apache.hadoop.mapred.InputFormat<NullWritable, DruidWritable>, VectorizedInputFormatInterface {
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidQueryBasedInputFormat.class);
-
- public static DruidQueryRecordReader getDruidQueryReader(String druidQueryType) {
- switch (druidQueryType) {
- case Query.TIMESERIES:
- return new DruidTimeseriesQueryRecordReader();
- case Query.TOPN:
- return new DruidTopNQueryRecordReader();
- case Query.GROUP_BY:
- return new DruidGroupByQueryRecordReader();
- case Query.SCAN:
- return new DruidScanQueryRecordReader();
- default:
- throw new IllegalStateException("Druid query type " + druidQueryType + " not recognized");
- }
- }
-
- @Override public org.apache.hadoop.mapred.InputSplit[] getSplits(JobConf job, int numSplits) throws IOException {
- return getInputSplits(job);
- }
-
- @Override public List<InputSplit> getSplits(JobContext context) throws IOException, InterruptedException {
- return Arrays.asList(getInputSplits(context.getConfiguration()));
- }
-
- protected HiveDruidSplit[] getInputSplits(Configuration conf) throws IOException {
- String address = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
- String queryId = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_QUERY_ID);
- if (StringUtils.isEmpty(address)) {
- throw new IOException("Druid broker address not specified in configuration");
- }
- String druidQuery = StringEscapeUtils.unescapeJava(conf.get(Constants.DRUID_QUERY_JSON));
-
- String druidQueryType;
- if (StringUtils.isEmpty(druidQuery)) {
- // Empty, maybe because CBO did not run; we fall back to
- // full Select query
- LOG.warn("Druid query is empty; creating Select query");
- String dataSource = conf.get(Constants.DRUID_DATA_SOURCE);
- if (dataSource == null || dataSource.isEmpty()) {
- throw new IOException("Druid data source cannot be empty or null");
- }
- druidQuery = DruidStorageHandlerUtils.createScanAllQuery(dataSource, Utilities.getColumnNames(conf));
- druidQueryType = Query.SCAN;
- conf.set(Constants.DRUID_QUERY_TYPE, druidQueryType);
- } else {
- druidQueryType = conf.get(Constants.DRUID_QUERY_TYPE);
- if (druidQueryType == null) {
- throw new IOException("Druid query type not recognized");
- }
- }
-
- // Add Hive Query ID to Druid Query
- if (queryId != null) {
- druidQuery = withQueryId(druidQuery, queryId);
- }
-
- // hive depends on FileSplits
- Job job = Job.getInstance(conf);
- JobContext jobContext = ShimLoader.getHadoopShims().newJobContext(job);
- Path[] paths = FileInputFormat.getInputPaths(jobContext);
-
- // We need to deserialize and serialize query so intervals are written in the JSON
- // Druid query with user timezone, as this is default Hive time semantics.
- // Then, create splits with the Druid queries.
- switch (druidQueryType) {
- case Query.TIMESERIES:
- case Query.TOPN:
- case Query.GROUP_BY:
- return new HiveDruidSplit[] {new HiveDruidSplit(druidQuery, paths[0], new String[] {address})};
- case Query.SCAN:
- ScanQuery scanQuery = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, ScanQuery.class);
- return distributeScanQuery(address, scanQuery, paths[0]);
- default:
- throw new IOException("Druid query type not recognized");
- }
- }
-
- /* New method that distributes the Scan query by creating splits containing
- * information about different Druid nodes that have the data for the given
- * query. */
- private static HiveDruidSplit[] distributeScanQuery(String address, ScanQuery query, Path dummyPath)
- throws IOException {
- // If it has a limit, we use it and we do not distribute the query
- final boolean isFetch = query.getScanRowsLimit() < Long.MAX_VALUE;
- if (isFetch) {
- return new HiveDruidSplit[] {new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(query),
- dummyPath,
- new String[] {address})
- };
- }
-
- final List<LocatedSegmentDescriptor> segmentDescriptors = fetchLocatedSegmentDescriptors(address, query);
-
- // Create one input split for each segment
- final int numSplits = segmentDescriptors.size();
- final HiveDruidSplit[] splits = new HiveDruidSplit[segmentDescriptors.size()];
- for (int i = 0; i < numSplits; i++) {
- final LocatedSegmentDescriptor locatedSD = segmentDescriptors.get(i);
- final String[] hosts = new String[locatedSD.getLocations().size() + 1];
- for (int j = 0; j < locatedSD.getLocations().size(); j++) {
- hosts[j] = locatedSD.getLocations().get(j).getHost();
- }
- // Default to broker if all other hosts fail.
- hosts[locatedSD.getLocations().size()] = address;
-
- // Create partial Select query
- final SegmentDescriptor
- newSD =
- new SegmentDescriptor(locatedSD.getInterval(), locatedSD.getVersion(), locatedSD.getPartitionNumber());
- final Query partialQuery = query.withQuerySegmentSpec(new MultipleSpecificSegmentSpec(Lists.newArrayList(newSD)));
- splits[i] =
- new HiveDruidSplit(DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(partialQuery), dummyPath, hosts);
- }
- return splits;
- }
-
- private static List<LocatedSegmentDescriptor> fetchLocatedSegmentDescriptors(String address, BaseQuery query)
- throws IOException {
- final String intervals = StringUtils.join(query.getIntervals(), ","); // Comma-separated intervals without brackets
- final String
- request =
- String.format("http://%s/druid/v2/datasources/%s/candidates?intervals=%s",
- address,
- query.getDataSource().getNames().get(0),
- URLEncoder.encode(intervals, "UTF-8"));
- LOG.debug("sending request {} to query for segments", request);
- final InputStream response;
- try {
- response =
- DruidStorageHandlerUtils.submitRequest(DruidStorageHandler.getHttpClient(),
- new Request(HttpMethod.GET, new URL(request)));
- } catch (Exception e) {
- throw new IOException(org.apache.hadoop.util.StringUtils.stringifyException(e));
- }
-
- // Retrieve results
- final List<LocatedSegmentDescriptor> segmentDescriptors;
- try {
- segmentDescriptors =
- DruidStorageHandlerUtils.JSON_MAPPER.readValue(response, new TypeReference<List<LocatedSegmentDescriptor>>() {
- });
- } catch (Exception e) {
- response.close();
- throw new IOException(org.apache.hadoop.util.StringUtils.stringifyException(e));
- }
- return segmentDescriptors;
- }
-
- private static String withQueryId(String druidQuery, String queryId) throws IOException {
- Query<?> queryWithId = DruidStorageHandlerUtils.JSON_MAPPER.readValue(druidQuery, BaseQuery.class).withId(queryId);
- return DruidStorageHandlerUtils.JSON_MAPPER.writeValueAsString(queryWithId);
- }
-
- @Override public org.apache.hadoop.mapred.RecordReader<NullWritable, DruidWritable> getRecordReader(
- org.apache.hadoop.mapred.InputSplit split,
- JobConf job,
- Reporter reporter) throws IOException {
- // We need to provide a different record reader for every type of Druid query.
- // The reason is that Druid results format is different for each type.
- final DruidQueryRecordReader<?> reader;
- // By default, we use druid scan query as fallback.
- final String druidQueryType = job.get(Constants.DRUID_QUERY_TYPE, Query.SCAN);
- reader = getDruidQueryReader(druidQueryType);
- reader.initialize((HiveDruidSplit) split, job);
- if (Utilities.getIsVectorized(job)) {
- //noinspection unchecked
- return (org.apache.hadoop.mapred.RecordReader) new DruidVectorizedWrapper(reader, job);
- }
- return reader;
- }
-
- @Override public RecordReader<NullWritable, DruidWritable> createRecordReader(InputSplit split,
- TaskAttemptContext context) throws IOException, InterruptedException {
- // By default, we use druid scan query as fallback.
- final String druidQueryType = context.getConfiguration().get(Constants.DRUID_QUERY_TYPE, Query.SCAN);
- // We need to provide a different record reader for every type of Druid query.
- // The reason is that Druid results format is different for each type.
- //noinspection unchecked
- return getDruidQueryReader(druidQueryType);
- }
-
- @Override public VectorizedSupport.Support[] getSupportedFeatures() {
- return new VectorizedSupport.Support[0];
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
deleted file mode 100644
index dc16c4e3f79..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidRecordWriter.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.io;
-
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Suppliers;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import org.apache.druid.data.input.Committer;
-import org.apache.druid.data.input.InputRow;
-import org.apache.druid.data.input.MapBasedInputRow;
-import org.apache.druid.java.util.common.DateTimes;
-import org.apache.druid.java.util.common.granularity.Granularity;
-import org.apache.druid.segment.indexing.DataSchema;
-import org.apache.druid.segment.indexing.RealtimeTuningConfig;
-import org.apache.druid.segment.loading.DataSegmentPusher;
-import org.apache.druid.segment.realtime.FireDepartmentMetrics;
-import org.apache.druid.segment.realtime.appenderator.Appenderator;
-import org.apache.druid.segment.realtime.appenderator.Appenderators;
-import org.apache.druid.segment.realtime.appenderator.SegmentIdWithShardSpec;
-import org.apache.druid.segment.realtime.appenderator.SegmentNotWritableException;
-import org.apache.druid.segment.realtime.appenderator.SegmentsAndMetadata;
-import org.apache.druid.segment.realtime.plumber.Committers;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.druid.serde.DruidWritable;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapred.RecordWriter;
-import org.apache.hadoop.mapred.Reporter;
-import org.joda.time.DateTime;
-import org.joda.time.Interval;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.ExecutionException;
-import java.util.function.Supplier;
-import java.util.stream.Collectors;
-
-/**
- * Druid Record Writer, implementing a file sink operator.
- */
-public class DruidRecordWriter implements RecordWriter<NullWritable, DruidWritable>, FileSinkOperator.RecordWriter {
- protected static final Logger LOG = LoggerFactory.getLogger(DruidRecordWriter.class);
-
- private final DataSchema dataSchema;
-
- private final Appenderator appenderator;
-
- private final RealtimeTuningConfig tuningConfig;
-
- private final Path segmentsDescriptorDir;
-
- private SegmentIdWithShardSpec currentOpenSegment = null;
-
- private final int maxPartitionSize;
-
- private final FileSystem fileSystem;
-
- private final Supplier<Committer> committerSupplier;
-
- private final Granularity segmentGranularity;
-
- public DruidRecordWriter(DataSchema dataSchema,
- RealtimeTuningConfig realtimeTuningConfig,
- DataSegmentPusher dataSegmentPusher,
- int maxPartitionSize,
- final Path segmentsDescriptorsDir,
- final FileSystem fileSystem) {
- File basePersistDir = new File(realtimeTuningConfig.getBasePersistDirectory(), UUID.randomUUID().toString());
- this.tuningConfig =
- Preconditions.checkNotNull(realtimeTuningConfig.withBasePersistDirectory(basePersistDir),
- "realtimeTuningConfig is null");
- this.dataSchema = Preconditions.checkNotNull(dataSchema, "data schema is null");
-
- appenderator = Appenderators
- .createOffline("hive-offline-appenderator", this.dataSchema, tuningConfig, false, new FireDepartmentMetrics(),
- dataSegmentPusher, DruidStorageHandlerUtils.JSON_MAPPER, DruidStorageHandlerUtils.INDEX_IO,
- DruidStorageHandlerUtils.INDEX_MERGER_V9);
- this.maxPartitionSize = maxPartitionSize;
- appenderator.startJob();
- this.segmentsDescriptorDir = Preconditions.checkNotNull(segmentsDescriptorsDir, "segmentsDescriptorsDir is null");
- this.fileSystem = Preconditions.checkNotNull(fileSystem, "file system is null");
- this.segmentGranularity = this.dataSchema.getGranularitySpec().getSegmentGranularity();
- committerSupplier = Suppliers.ofInstance(Committers.nil())::get;
- }
-
- /**
- * This function computes the segment identifier and push the current open segment
- * The push will occur if max size is reached or the event belongs to the next interval.
- * Note that this function assumes that timestamps are pseudo sorted.
- * This function will close and move to the next segment granularity as soon as
- * an event from the next interval appears. The sorting is done by the previous stage.
- *
- * @return segmentIdentifier with of the truncatedTime and maybe push the current open segment.
- */
- private SegmentIdWithShardSpec getSegmentIdentifierAndMaybePush(long truncatedTime) {
-
- DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(truncatedTime));
- final Interval interval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime));
-
- SegmentIdWithShardSpec retVal;
- if (currentOpenSegment == null) {
- currentOpenSegment =
- new SegmentIdWithShardSpec(dataSchema.getDataSource(),
- interval,
- tuningConfig.getVersioningPolicy().getVersion(interval),
- new LinearShardSpec(0));
- return currentOpenSegment;
- } else if (currentOpenSegment.getInterval().equals(interval)) {
- retVal = currentOpenSegment;
- int rowCount = appenderator.getRowCount(retVal);
- if (rowCount < maxPartitionSize) {
- return retVal;
- } else {
- retVal =
- new SegmentIdWithShardSpec(dataSchema.getDataSource(),
- interval,
- tuningConfig.getVersioningPolicy().getVersion(interval),
- new LinearShardSpec(currentOpenSegment.getShardSpec().getPartitionNum() + 1));
- pushSegments(Lists.newArrayList(currentOpenSegment));
- LOG.info("Creating new partition for segment {}, partition num {}",
- retVal.toString(),
- retVal.getShardSpec().getPartitionNum());
- currentOpenSegment = retVal;
- return retVal;
- }
- } else {
- retVal =
- new SegmentIdWithShardSpec(dataSchema.getDataSource(),
- interval,
- tuningConfig.getVersioningPolicy().getVersion(interval),
- new LinearShardSpec(0));
- pushSegments(Lists.newArrayList(currentOpenSegment));
- LOG.info("Creating segment {}", retVal.toString());
- currentOpenSegment = retVal;
- return retVal;
- }
- }
-
- private void pushSegments(List<SegmentIdWithShardSpec> segmentsToPush) {
- try {
- SegmentsAndMetadata segmentsAndMetadata = appenderator.push(segmentsToPush, committerSupplier.get(), false).get();
- final Set<String> pushedSegmentIdentifierHashSet = new HashSet<>();
-
- for (DataSegment pushedSegment : segmentsAndMetadata.getSegments()) {
- pushedSegmentIdentifierHashSet.add(SegmentIdWithShardSpec.fromDataSegment(pushedSegment).toString());
- final Path
- segmentDescriptorOutputPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(pushedSegment, segmentsDescriptorDir);
- DruidStorageHandlerUtils.writeSegmentDescriptor(fileSystem, pushedSegment, segmentDescriptorOutputPath);
- LOG.info(String.format("Pushed the segment [%s] and persisted the descriptor located at [%s]",
- pushedSegment,
- segmentDescriptorOutputPath));
- }
-
- final Set<String>
- toPushSegmentsHashSet =
- segmentsToPush.stream()
- .map(SegmentIdWithShardSpec::toString)
- .collect(Collectors.toCollection(HashSet::new));
-
- if (!pushedSegmentIdentifierHashSet.equals(toPushSegmentsHashSet)) {
- throw new IllegalStateException(String.format("was asked to publish [%s] but was able to publish only [%s]",
- Joiner.on(", ").join(toPushSegmentsHashSet),
- Joiner.on(", ").join(pushedSegmentIdentifierHashSet)));
- }
- for (SegmentIdWithShardSpec dataSegmentId : segmentsToPush) {
- LOG.info("Dropping segment {}", dataSegmentId.toString());
- appenderator.drop(dataSegmentId).get();
- }
-
- LOG.info(String.format("Published [%,d] segments.", segmentsToPush.size()));
- } catch (InterruptedException e) {
- LOG.error(String.format("got interrupted, failed to push [%,d] segments.", segmentsToPush.size()), e);
- Thread.currentThread().interrupt();
- } catch (IOException | ExecutionException e) {
- LOG.error(String.format("Failed to push [%,d] segments.", segmentsToPush.size()), e);
- Throwables.propagate(e);
- }
- }
-
- @Override public void write(Writable w) throws IOException {
- DruidWritable record = (DruidWritable) w;
- final long timestamp = (long) record.getValue().get(DruidConstants.DEFAULT_TIMESTAMP_COLUMN);
- final int
- partitionNumber =
- Math.toIntExact((long) record.getValue().getOrDefault(Constants.DRUID_SHARD_KEY_COL_NAME, -1L));
- final InputRow
- inputRow =
- new MapBasedInputRow(timestamp,
- dataSchema.getParser().getParseSpec().getDimensionsSpec().getDimensionNames(),
- record.getValue());
-
- try {
-
- if (partitionNumber != -1 && maxPartitionSize == -1) {
- /*
- Case data is sorted by time and an extra hashing dimension see DRUID_SHARD_KEY_COL_NAME
- Thus use DRUID_SHARD_KEY_COL_NAME as segment partition in addition to time dimension
- Data with the same DRUID_SHARD_KEY_COL_NAME and Time interval will end in the same segment
- */
- DateTime truncatedDateTime = segmentGranularity.bucketStart(DateTimes.utc(timestamp));
- final Interval interval = new Interval(truncatedDateTime, segmentGranularity.increment(truncatedDateTime));
-
- if (currentOpenSegment != null) {
- if (currentOpenSegment.getShardSpec().getPartitionNum() != partitionNumber
- || !currentOpenSegment.getInterval().equals(interval)) {
- pushSegments(ImmutableList.of(currentOpenSegment));
- currentOpenSegment =
- new SegmentIdWithShardSpec(dataSchema.getDataSource(),
- interval,
- tuningConfig.getVersioningPolicy().getVersion(interval),
- new LinearShardSpec(partitionNumber));
- }
- } else {
- currentOpenSegment =
- new SegmentIdWithShardSpec(dataSchema.getDataSource(),
- interval,
- tuningConfig.getVersioningPolicy().getVersion(interval),
- new LinearShardSpec(partitionNumber));
-
- }
- appenderator.add(currentOpenSegment, inputRow, committerSupplier::get);
-
- } else if (partitionNumber == -1 && maxPartitionSize != -1) {
- /*Case we are partitioning the segments based on time and max row per segment maxPartitionSize*/
- appenderator.add(getSegmentIdentifierAndMaybePush(timestamp), inputRow, committerSupplier::get);
- } else {
- throw new IllegalArgumentException(String.format(
- "partitionNumber and maxPartitionSize should be mutually exclusive "
- + "got partitionNum [%s] and maxPartitionSize [%s]",
- partitionNumber,
- maxPartitionSize));
- }
-
- } catch (SegmentNotWritableException e) {
- throw new IOException(e);
- }
- }
-
- @Override public void close(boolean abort) throws IOException {
- try {
- if (!abort) {
- final List<SegmentIdWithShardSpec> segmentsToPush = Lists.newArrayList();
- segmentsToPush.addAll(appenderator.getSegments());
- pushSegments(segmentsToPush);
- }
- appenderator.clear();
- } catch (InterruptedException e) {
- Throwables.propagate(e);
- } finally {
- try {
- FileUtils.deleteDirectory(tuningConfig.getBasePersistDirectory());
- } catch (Exception e) {
- LOG.error("error cleaning of base persist directory", e);
- }
- appenderator.close();
- }
- }
-
- @Override public void write(NullWritable key, DruidWritable value) throws IOException {
- this.write(value);
- }
-
- @Override public void close(Reporter reporter) throws IOException {
- this.close(false);
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java
deleted file mode 100644
index 2345e13ab07..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/DruidVectorizedWrapper.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.io;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader;
-import org.apache.hadoop.hive.druid.serde.DruidSerDe;
-import org.apache.hadoop.hive.druid.serde.DruidWritable;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.exec.vector.VectorAssignRow;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
-import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.ql.plan.MapWork;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapred.RecordReader;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.Properties;
-
-/**
- * A Wrapper class that consumes row-by-row from base Druid Record Reader and provides a Vectorized one.
- * @param <T> type of the Druid query.
- */
-public class DruidVectorizedWrapper<T extends Comparable<T>> implements RecordReader<NullWritable, VectorizedRowBatch> {
- private final VectorAssignRow vectorAssignRow = new VectorAssignRow();
- private final DruidQueryRecordReader baseReader;
- private final VectorizedRowBatchCtx rbCtx;
- private final DruidSerDe serDe;
- private final Object[] rowBoat;
-
- /**
- * Actual projected columns needed by the query, this can be empty in case of query like: select count(*) from src.
- */
- private final int[] projectedColumns;
-
- private final DruidWritable druidWritable;
-
- public DruidVectorizedWrapper(DruidQueryRecordReader reader, Configuration jobConf) {
- this.rbCtx = Utilities.getVectorizedRowBatchCtx(jobConf);
- if (rbCtx.getDataColumnNums() != null) {
- projectedColumns = rbCtx.getDataColumnNums();
- } else {
- // case all the columns are selected
- projectedColumns = new int[rbCtx.getRowColumnTypeInfos().length];
- for (int i = 0; i < projectedColumns.length; i++) {
- projectedColumns[i] = i;
- }
- }
- this.serDe = createAndInitializeSerde(jobConf);
- this.baseReader = Preconditions.checkNotNull(reader);
-
- // row parser and row assigner initializing
- try {
- vectorAssignRow.init((StructObjectInspector) serDe.getObjectInspector());
- } catch (HiveException e) {
- throw new RuntimeException(e);
- }
-
- druidWritable = baseReader.createValue();
- rowBoat = new Object[rbCtx.getDataColumnCount()];
- }
-
- @Override public boolean next(NullWritable nullWritable, VectorizedRowBatch vectorizedRowBatch) throws IOException {
- vectorizedRowBatch.reset();
- int rowsCount = 0;
- while (rowsCount < vectorizedRowBatch.getMaxSize() && baseReader.next(nullWritable, druidWritable)) {
- if (projectedColumns.length > 0) {
- try {
- serDe.deserializeAsPrimitive(druidWritable, rowBoat);
- } catch (SerDeException e) {
- throw new IOException(e);
- }
- for (int i : projectedColumns) {
- vectorAssignRow.assignRowColumn(vectorizedRowBatch, rowsCount, i, rowBoat[i]);
- }
- }
- rowsCount++;
- }
- vectorizedRowBatch.size = rowsCount;
- return rowsCount > 0;
- }
-
- @Override public NullWritable createKey() {
- return NullWritable.get();
- }
-
- @Override public VectorizedRowBatch createValue() {
- return rbCtx.createVectorizedRowBatch();
- }
-
- @Override public long getPos() throws IOException {
- return baseReader.getPos();
- }
-
- @Override public void close() throws IOException {
- baseReader.close();
- }
-
- @Override public float getProgress() throws IOException {
- return baseReader.getProgress();
- }
-
- private static DruidSerDe createAndInitializeSerde(Configuration jobConf) {
- DruidSerDe serDe = new DruidSerDe();
- MapWork mapWork = Preconditions.checkNotNull(Utilities.getMapWork(jobConf), "Map work is null");
- Iterator<org.apache.hadoop.hive.ql.plan.PartitionDesc> partitionIterator = mapWork.getPartitionDescs();
- if (!partitionIterator.hasNext()) {
- throw new RuntimeException("Can not find table property at the map work");
- }
- Properties properties = partitionIterator.next().getTableDesc().getProperties();
- try {
- serDe.initialize(jobConf, properties, null);
- } catch (SerDeException e) {
- throw new RuntimeException("Can not initialized the serde", e);
- }
- return serDe;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java
deleted file mode 100644
index 89d74e38bfc..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/HiveDruidSplit.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.io;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.Arrays;
-
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileSplit;
-
-/**
- * Druid split. Its purpose is to trigger query execution in Druid.
- */
-public class HiveDruidSplit extends FileSplit implements org.apache.hadoop.mapred.InputSplit {
-
- private String druidQuery;
-
- private String[] hosts;
-
- // required for deserialization
- public HiveDruidSplit() {
- super(null, 0, 0, (String[]) null);
- }
-
- public HiveDruidSplit(String druidQuery, Path dummyPath, String[] hosts) {
- super(dummyPath, 0, 0, hosts);
- this.druidQuery = druidQuery;
- this.hosts = hosts;
- }
-
- @Override public void write(DataOutput out) throws IOException {
- super.write(out);
- out.writeUTF(druidQuery);
- out.writeInt(hosts.length);
- for (String host : hosts) {
- out.writeUTF(host);
- }
- }
-
- @Override public void readFields(DataInput in) throws IOException {
- super.readFields(in);
- druidQuery = in.readUTF();
- int length = in.readInt();
- String[] listHosts = new String[length];
- for (int i = 0; i < length; i++) {
- listHosts[i] = in.readUTF();
- }
- hosts = listHosts;
- }
-
- public String getDruidQuery() {
- return druidQuery;
- }
-
- @Override public String[] getLocations() throws IOException {
- return hosts;
- }
-
- @Override public String toString() {
- return "HiveDruidSplit{" + druidQuery + ", " + (hosts == null ? "empty hosts" : Arrays.toString(hosts)) + "}";
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java
deleted file mode 100644
index e5aac26f1bc..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/io/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package info file.
- */
-
-package org.apache.hadoop.hive.druid.io;
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java
deleted file mode 100644
index 199987a9c10..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroBytesDecoder.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes(value = {
- @JsonSubTypes.Type(name = "schema_inline", value = InlineSchemaAvroBytesDecoder.class) })
-public interface AvroBytesDecoder {
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java
deleted file mode 100644
index 48d6cf2d6c8..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroParseSpec.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.druid.data.input.impl.DimensionsSpec;
-import org.apache.druid.data.input.impl.ParseSpec;
-import org.apache.druid.data.input.impl.TimestampSpec;
-import org.apache.druid.java.util.common.parsers.JSONPathSpec;
-import org.apache.druid.java.util.common.parsers.Parser;
-
-import java.util.Objects;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class AvroParseSpec extends ParseSpec {
-
- @JsonIgnore private final JSONPathSpec flattenSpec;
-
- @JsonCreator public AvroParseSpec(@JsonProperty("timestampSpec") TimestampSpec timestampSpec,
- @JsonProperty("dimensionsSpec") DimensionsSpec dimensionsSpec,
- @JsonProperty("flattenSpec") JSONPathSpec flattenSpec) {
- super(timestampSpec != null ? timestampSpec : new TimestampSpec(null, null, null),
- dimensionsSpec != null ? dimensionsSpec : new DimensionsSpec(null, null, null));
-
- this.flattenSpec = flattenSpec != null ? flattenSpec : JSONPathSpec.DEFAULT;
- }
-
- @JsonProperty public JSONPathSpec getFlattenSpec() {
- return flattenSpec;
- }
-
- @Override public Parser<String, Object> makeParser() {
- // makeParser is only used by StringInputRowParser, which cannot parse avro anyway.
- throw new UnsupportedOperationException("makeParser not supported");
- }
-
- @Override public ParseSpec withTimestampSpec(TimestampSpec spec) {
- return new AvroParseSpec(spec, getDimensionsSpec(), flattenSpec);
- }
-
- @Override public ParseSpec withDimensionsSpec(DimensionsSpec spec) {
- return new AvroParseSpec(getTimestampSpec(), spec, flattenSpec);
- }
-
- @Override public boolean equals(final Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- final AvroParseSpec that = (AvroParseSpec) o;
- return Objects.equals(flattenSpec, that.flattenSpec);
- }
-
- @Override public int hashCode() {
- return Objects.hash(super.hashCode(), flattenSpec);
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java
deleted file mode 100644
index b689b63031a..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/AvroStreamInputRowParser.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-import org.apache.druid.data.input.ByteBufferInputRowParser;
-import org.apache.druid.data.input.InputRow;
-import org.apache.druid.data.input.impl.ParseSpec;
-
-import javax.validation.constraints.NotNull;
-import java.nio.ByteBuffer;
-import java.util.List;
-import java.util.Objects;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class AvroStreamInputRowParser implements ByteBufferInputRowParser {
- private final ParseSpec parseSpec;
- private final AvroBytesDecoder avroBytesDecoder;
-
- @JsonCreator public AvroStreamInputRowParser(@JsonProperty("parseSpec") ParseSpec parseSpec,
- @JsonProperty("avroBytesDecoder") AvroBytesDecoder avroBytesDecoder) {
- this.parseSpec = Preconditions.checkNotNull(parseSpec, "parseSpec");
- this.avroBytesDecoder = Preconditions.checkNotNull(avroBytesDecoder, "avroBytesDecoder");
- }
-
- @NotNull @Override public List<InputRow> parseBatch(ByteBuffer input) {
- throw new UnsupportedOperationException("This class is only used for JSON serde");
- }
-
- @JsonProperty @Override public ParseSpec getParseSpec() {
- return parseSpec;
- }
-
- @JsonProperty public AvroBytesDecoder getAvroBytesDecoder() {
- return avroBytesDecoder;
- }
-
- @Override public ByteBufferInputRowParser withParseSpec(ParseSpec parseSpec) {
- return new AvroStreamInputRowParser(parseSpec, avroBytesDecoder);
- }
-
- @Override public boolean equals(final Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- final AvroStreamInputRowParser that = (AvroStreamInputRowParser) o;
- return Objects.equals(parseSpec, that.parseSpec) && Objects.equals(avroBytesDecoder, that.avroBytesDecoder);
- }
-
- @Override public int hashCode() {
- return Objects.hash(parseSpec, avroBytesDecoder);
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java
deleted file mode 100644
index 1b6a08e7466..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/InlineSchemaAvroBytesDecoder.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-
-import java.util.Map;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class InlineSchemaAvroBytesDecoder implements AvroBytesDecoder {
- private final Map<String, Object> schema;
-
- @JsonCreator public InlineSchemaAvroBytesDecoder(@JsonProperty("schema") Map<String, Object> schema) {
- Preconditions.checkArgument(schema != null, "schema must be provided");
-
- this.schema = schema;
- }
-
- @JsonProperty public Map<String, Object> getSchema() {
- return schema;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java
deleted file mode 100644
index 92800cef5fd..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaIndexTaskTuningConfig.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
-import org.joda.time.Period;
-
-import javax.annotation.Nullable;
-import java.io.File;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class KafkaIndexTaskTuningConfig extends SeekableStreamIndexTaskTuningConfig {
- @JsonCreator
- public KafkaIndexTaskTuningConfig(
- @JsonProperty("maxRowsInMemory") @Nullable Integer maxRowsInMemory,
- @JsonProperty("maxBytesInMemory") @Nullable Long maxBytesInMemory,
- @JsonProperty("maxRowsPerSegment") @Nullable Integer maxRowsPerSegment,
- @JsonProperty("maxTotalRows") @Nullable Long maxTotalRows,
- @JsonProperty("intermediatePersistPeriod") @Nullable Period intermediatePersistPeriod,
- @JsonProperty("basePersistDirectory") @Nullable File basePersistDirectory,
- @JsonProperty("maxPendingPersists") @Nullable Integer maxPendingPersists,
- @JsonProperty("indexSpec") @Nullable IndexSpec indexSpec,
- @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
- // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
- @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
- @Deprecated @JsonProperty("reportParseExceptions") @Nullable Boolean reportParseExceptions,
- @JsonProperty("handoffConditionTimeout") @Nullable Long handoffConditionTimeout,
- @JsonProperty("resetOffsetAutomatically") @Nullable Boolean resetOffsetAutomatically,
- @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
- @JsonProperty("intermediateHandoffPeriod") @Nullable Period intermediateHandoffPeriod,
- @JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions,
- @JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
- @JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions
- ) {
- super(
- maxRowsInMemory,
- maxBytesInMemory,
- maxRowsPerSegment,
- maxTotalRows,
- intermediatePersistPeriod,
- basePersistDirectory,
- maxPendingPersists,
- indexSpec,
- indexSpecForIntermediatePersists,
- true,
- reportParseExceptions,
- handoffConditionTimeout,
- resetOffsetAutomatically,
- false,
- segmentWriteOutMediumFactory,
- intermediateHandoffPeriod,
- logParseExceptions,
- maxParseExceptions,
- maxSavedParseExceptions
- );
- }
-
- @Override
- public KafkaIndexTaskTuningConfig withBasePersistDirectory(File dir) {
- return new KafkaIndexTaskTuningConfig(
- getMaxRowsInMemory(),
- getMaxBytesInMemory(),
- getMaxRowsPerSegment(),
- getMaxTotalRows(),
- getIntermediatePersistPeriod(),
- dir,
- getMaxPendingPersists(),
- getIndexSpec(),
- getIndexSpecForIntermediatePersists(),
- true,
- isReportParseExceptions(),
- getHandoffConditionTimeout(),
- isResetOffsetAutomatically(),
- getSegmentWriteOutMediumFactory(),
- getIntermediateHandoffPeriod(),
- isLogParseExceptions(),
- getMaxParseExceptions(),
- getMaxSavedParseExceptions()
- );
- }
-
-
- @Override
- public String toString() {
- return "KafkaIndexTaskTuningConfig{" +
- "maxRowsInMemory=" + getMaxRowsInMemory() +
- ", maxRowsPerSegment=" + getMaxRowsPerSegment() +
- ", maxTotalRows=" + getMaxTotalRows() +
- ", maxBytesInMemory=" + getMaxBytesInMemory() +
- ", intermediatePersistPeriod=" + getIntermediatePersistPeriod() +
- ", basePersistDirectory=" + getBasePersistDirectory() +
- ", maxPendingPersists=" + getMaxPendingPersists() +
- ", indexSpec=" + getIndexSpec() +
- ", indexSpecForIntermediatePersists=" + getIndexSpecForIntermediatePersists() +
- ", reportParseExceptions=" + isReportParseExceptions() +
- ", handoffConditionTimeout=" + getHandoffConditionTimeout() +
- ", resetOffsetAutomatically=" + isResetOffsetAutomatically() +
- ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() +
- ", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() +
- ", logParseExceptions=" + isLogParseExceptions() +
- ", maxParseExceptions=" + getMaxParseExceptions() +
- ", maxSavedParseExceptions=" + getMaxSavedParseExceptions() +
- '}';
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java
deleted file mode 100644
index 9f5fc48e6e6..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorIOConfig.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import org.apache.druid.java.util.common.StringUtils;
-import org.joda.time.Duration;
-import org.joda.time.Period;
-
-import java.util.Map;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class KafkaSupervisorIOConfig {
- public static final String BOOTSTRAP_SERVERS_KEY = "bootstrap.servers";
-
- private final String topic;
- private final Integer replicas;
- private final Integer taskCount;
- private final Duration taskDuration;
- private final Map<String, String> consumerProperties;
- private final Duration startDelay;
- private final Duration period;
- private final boolean useEarliestOffset;
- private final Duration completionTimeout;
- @SuppressWarnings({ "OptionalUsedAsFieldOrParameterType", "Guava" }) private final Optional<Duration>
- lateMessageRejectionPeriod;
- @SuppressWarnings({ "OptionalUsedAsFieldOrParameterType", "Guava" }) private final Optional<Duration>
- earlyMessageRejectionPeriod;
- private final boolean skipOffsetGaps;
-
- @JsonCreator public KafkaSupervisorIOConfig(@JsonProperty("topic") String topic,
- @JsonProperty("replicas") Integer replicas,
- @JsonProperty("taskCount") Integer taskCount,
- @JsonProperty("taskDuration") Period taskDuration,
- @JsonProperty("consumerProperties") Map<String, String> consumerProperties,
- @JsonProperty("startDelay") Period startDelay,
- @JsonProperty("period") Period period,
- @JsonProperty("useEarliestOffset") Boolean useEarliestOffset,
- @JsonProperty("completionTimeout") Period completionTimeout,
- @JsonProperty("lateMessageRejectionPeriod") Period lateMessageRejectionPeriod,
- @JsonProperty("earlyMessageRejectionPeriod") Period earlyMessageRejectionPeriod,
- @JsonProperty("skipOffsetGaps") Boolean skipOffsetGaps) {
- this.topic = Preconditions.checkNotNull(topic, "topic");
- this.consumerProperties = Preconditions.checkNotNull(consumerProperties, "consumerProperties");
- Preconditions.checkNotNull(consumerProperties.get(BOOTSTRAP_SERVERS_KEY),
- StringUtils.format("consumerProperties must contain entry for [%s]", BOOTSTRAP_SERVERS_KEY));
-
- this.replicas = replicas != null ? replicas : 1;
- this.taskCount = taskCount != null ? taskCount : 1;
- this.taskDuration = defaultDuration(taskDuration, "PT1H");
- this.startDelay = defaultDuration(startDelay, "PT5S");
- this.period = defaultDuration(period, "PT30S");
- this.useEarliestOffset = useEarliestOffset != null ? useEarliestOffset : false;
- this.completionTimeout = defaultDuration(completionTimeout, "PT30M");
- //noinspection Guava
- this.lateMessageRejectionPeriod =
- lateMessageRejectionPeriod == null ?
- Optional.absent() :
- Optional.of(lateMessageRejectionPeriod.toStandardDuration());
- //noinspection Guava
- this.earlyMessageRejectionPeriod =
- earlyMessageRejectionPeriod == null ?
- Optional.absent() :
- Optional.of(earlyMessageRejectionPeriod.toStandardDuration());
- this.skipOffsetGaps = skipOffsetGaps != null ? skipOffsetGaps : false;
- }
-
- @JsonProperty public String getTopic() {
- return topic;
- }
-
- @JsonProperty public Integer getReplicas() {
- return replicas;
- }
-
- @JsonProperty public Integer getTaskCount() {
- return taskCount;
- }
-
- @JsonProperty public Duration getTaskDuration() {
- return taskDuration;
- }
-
- @JsonProperty public Map<String, String> getConsumerProperties() {
- return consumerProperties;
- }
-
- @JsonProperty public Duration getStartDelay() {
- return startDelay;
- }
-
- @JsonProperty public Duration getPeriod() {
- return period;
- }
-
- @JsonProperty public boolean isUseEarliestOffset() {
- return useEarliestOffset;
- }
-
- @JsonProperty public Duration getCompletionTimeout() {
- return completionTimeout;
- }
-
- @SuppressWarnings("Guava") @JsonProperty public Optional<Duration> getEarlyMessageRejectionPeriod() {
- return earlyMessageRejectionPeriod;
- }
-
- @SuppressWarnings("Guava") @JsonProperty public Optional<Duration> getLateMessageRejectionPeriod() {
- return lateMessageRejectionPeriod;
- }
-
- @JsonProperty public boolean isSkipOffsetGaps() {
- return skipOffsetGaps;
- }
-
- @Override public String toString() {
- return "KafkaSupervisorIOConfig{"
- + "topic='"
- + topic
- + '\''
- + ", replicas="
- + replicas
- + ", taskCount="
- + taskCount
- + ", taskDuration="
- + taskDuration
- + ", consumerProperties="
- + consumerProperties
- + ", startDelay="
- + startDelay
- + ", period="
- + period
- + ", useEarliestOffset="
- + useEarliestOffset
- + ", completionTimeout="
- + completionTimeout
- + ", lateMessageRejectionPeriod="
- + lateMessageRejectionPeriod
- + ", skipOffsetGaps="
- + skipOffsetGaps
- + '}';
- }
-
- private static Duration defaultDuration(final Period period, final String theDefault) {
- return (period == null ? new Period(theDefault) : period).toStandardDuration();
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- KafkaSupervisorIOConfig that = (KafkaSupervisorIOConfig) o;
-
- if (useEarliestOffset != that.useEarliestOffset) {
- return false;
- }
- if (skipOffsetGaps != that.skipOffsetGaps) {
- return false;
- }
- if (topic != null ? !topic.equals(that.topic) : that.topic != null) {
- return false;
- }
- if (replicas != null ? !replicas.equals(that.replicas) : that.replicas != null) {
- return false;
- }
- if (taskCount != null ? !taskCount.equals(that.taskCount) : that.taskCount != null) {
- return false;
- }
- if (taskDuration != null ? !taskDuration.equals(that.taskDuration) : that.taskDuration != null) {
- return false;
- }
- if (consumerProperties != null ?
- !consumerProperties.equals(that.consumerProperties) :
- that.consumerProperties != null) {
- return false;
- }
- if (startDelay != null ? !startDelay.equals(that.startDelay) : that.startDelay != null) {
- return false;
- }
- if (period != null ? !period.equals(that.period) : that.period != null) {
- return false;
- }
- if (completionTimeout != null ?
- !completionTimeout.equals(that.completionTimeout) :
- that.completionTimeout != null) {
- return false;
- }
- if (lateMessageRejectionPeriod.isPresent() ?
- !lateMessageRejectionPeriod.equals(that.lateMessageRejectionPeriod) :
- that.lateMessageRejectionPeriod.isPresent()) {
- return false;
- }
- return earlyMessageRejectionPeriod.isPresent() ?
- earlyMessageRejectionPeriod.equals(that.earlyMessageRejectionPeriod) :
- !that.earlyMessageRejectionPeriod.isPresent();
- }
-
- @Override public int hashCode() {
- int result = topic != null ? topic.hashCode() : 0;
- result = 31 * result + (replicas != null ? replicas.hashCode() : 0);
- result = 31 * result + (taskCount != null ? taskCount.hashCode() : 0);
- result = 31 * result + (taskDuration != null ? taskDuration.hashCode() : 0);
- result = 31 * result + (consumerProperties != null ? consumerProperties.hashCode() : 0);
- result = 31 * result + (startDelay != null ? startDelay.hashCode() : 0);
- result = 31 * result + (period != null ? period.hashCode() : 0);
- result = 31 * result + (useEarliestOffset ? 1 : 0);
- result = 31 * result + (completionTimeout != null ? completionTimeout.hashCode() : 0);
- result = 31 * result + (lateMessageRejectionPeriod.isPresent() ? lateMessageRejectionPeriod.hashCode() : 0);
- result = 31 * result + (earlyMessageRejectionPeriod.isPresent() ? earlyMessageRejectionPeriod.hashCode() : 0);
- result = 31 * result + (skipOffsetGaps ? 1 : 0);
- return result;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java
deleted file mode 100644
index bca1c789684..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorReport.java
+++ /dev/null
@@ -1,202 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.collect.Lists;
-import org.apache.druid.indexing.overlord.supervisor.SupervisorReport;
-import org.joda.time.DateTime;
-
-import javax.annotation.Nullable;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public class KafkaSupervisorReport extends SupervisorReport {
- /**
- * Report Payload class.
- */
- public static class KafkaSupervisorReportPayload {
- private final String dataSource;
- private final String topic;
- private final Integer partitions;
- private final Integer replicas;
- private final Long durationSeconds;
- private final List<TaskReportData> activeTasks;
- private final List<TaskReportData> publishingTasks;
- private final Map<Integer, Long> latestOffsets;
- private final Map<Integer, Long> minimumLag;
- private final Long aggregateLag;
- private final DateTime offsetsLastUpdated;
-
- @JsonCreator public KafkaSupervisorReportPayload(@JsonProperty("dataSource") String dataSource,
- @JsonProperty("topic") String topic,
- @JsonProperty("partitions") Integer partitions,
- @JsonProperty("replicas") Integer replicas,
- @JsonProperty("durationSeconds") Long durationSeconds,
- @Nullable @JsonProperty("latestOffsets") Map<Integer, Long> latestOffsets,
- @Nullable @JsonProperty("minimumLag") Map<Integer, Long> minimumLag,
- @Nullable @JsonProperty("aggregateLag") Long aggregateLag,
- @Nullable @JsonProperty("offsetsLastUpdated") DateTime offsetsLastUpdated) {
- this.dataSource = dataSource;
- this.topic = topic;
- this.partitions = partitions;
- this.replicas = replicas;
- this.durationSeconds = durationSeconds;
- this.activeTasks = Lists.newArrayList();
- this.publishingTasks = Lists.newArrayList();
- this.latestOffsets = latestOffsets;
- this.minimumLag = minimumLag;
- this.aggregateLag = aggregateLag;
- this.offsetsLastUpdated = offsetsLastUpdated;
- }
-
- @JsonProperty public String getDataSource() {
- return dataSource;
- }
-
- @JsonProperty public String getTopic() {
- return topic;
- }
-
- @JsonProperty public Integer getPartitions() {
- return partitions;
- }
-
- @JsonProperty public Integer getReplicas() {
- return replicas;
- }
-
- @JsonProperty public Long getDurationSeconds() {
- return durationSeconds;
- }
-
- @JsonProperty public List<TaskReportData> getActiveTasks() {
- return activeTasks;
- }
-
- @JsonProperty public List<TaskReportData> getPublishingTasks() {
- return publishingTasks;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map<Integer, Long> getLatestOffsets() {
- return latestOffsets;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map<Integer, Long> getMinimumLag() {
- return minimumLag;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Long getAggregateLag() {
- return aggregateLag;
- }
-
- @JsonProperty public DateTime getOffsetsLastUpdated() {
- return offsetsLastUpdated;
- }
-
- @Override public String toString() {
- return "{"
- + "dataSource='"
- + dataSource
- + '\''
- + ", topic='"
- + topic
- + '\''
- + ", partitions="
- + partitions
- + ", replicas="
- + replicas
- + ", durationSeconds="
- + durationSeconds
- + ", active="
- + activeTasks
- + ", publishing="
- + publishingTasks
- + (latestOffsets != null ? ", latestOffsets=" + latestOffsets : "")
- + (minimumLag != null ? ", minimumLag=" + minimumLag : "")
- + (aggregateLag != null ? ", aggregateLag=" + aggregateLag : "")
- + (offsetsLastUpdated != null ? ", offsetsLastUpdated=" + offsetsLastUpdated : "")
- + '}';
- }
- }
-
- private final KafkaSupervisorReportPayload payload;
-
- @JsonCreator public KafkaSupervisorReport(@JsonProperty("id") String id,
- @JsonProperty("generationTime") DateTime generationTime,
- @JsonProperty("payload") KafkaSupervisorReportPayload payload) {
- super(id, generationTime, payload);
- this.payload = payload;
- }
-
- public KafkaSupervisorReport(String dataSource,
- DateTime generationTime,
- String topic,
- Integer partitions,
- Integer replicas,
- Long durationSeconds,
- @Nullable Map<Integer, Long> latestOffsets,
- @Nullable Map<Integer, Long> minimumLag,
- @Nullable Long aggregateLag,
- @Nullable DateTime offsetsLastUpdated) {
- this(dataSource,
- generationTime,
- new KafkaSupervisorReportPayload(dataSource,
- topic,
- partitions,
- replicas,
- durationSeconds,
- latestOffsets,
- minimumLag,
- aggregateLag,
- offsetsLastUpdated));
- }
-
- @Override public KafkaSupervisorReportPayload getPayload() {
- return payload;
- }
-
- @Override public String toString() {
- return "{" + "id='" + getId() + '\'' + ", generationTime=" + getGenerationTime() + ", payload=" + payload + '}';
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- if (!super.equals(o)) {
- return false;
- }
- KafkaSupervisorReport that = (KafkaSupervisorReport) o;
- return Objects.equals(payload, that.payload);
- }
-
- @Override public int hashCode() {
- return Objects.hash(super.hashCode(), payload);
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java
deleted file mode 100644
index 18177b6640c..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorSpec.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import org.apache.druid.segment.indexing.DataSchema;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import com.google.common.base.Preconditions;
-
-import java.util.Map;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type") @JsonSubTypes({
- @JsonSubTypes.Type(name = "kafka", value = KafkaSupervisorSpec.class) }) public class KafkaSupervisorSpec {
- private final DataSchema dataSchema;
- private final KafkaSupervisorTuningConfig tuningConfig;
- private final KafkaSupervisorIOConfig ioConfig;
- private final Map<String, Object> context;
-
- @JsonCreator public KafkaSupervisorSpec(@JsonProperty("dataSchema") DataSchema dataSchema,
- @JsonProperty("tuningConfig") KafkaSupervisorTuningConfig tuningConfig,
- @JsonProperty("ioConfig") KafkaSupervisorIOConfig ioConfig,
- @JsonProperty("context") Map<String, Object> context) {
- this.dataSchema = Preconditions.checkNotNull(dataSchema, "dataSchema");
- this.tuningConfig =
- tuningConfig != null ?
- tuningConfig :
- new KafkaSupervisorTuningConfig(null,
- null,
- null,
- null,
- null,
- null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, null);
- this.ioConfig = Preconditions.checkNotNull(ioConfig, "ioConfig");
- this.context = context;
- }
-
- @JsonProperty public DataSchema getDataSchema() {
- return dataSchema;
- }
-
- @JsonProperty public KafkaSupervisorTuningConfig getTuningConfig() {
- return tuningConfig;
- }
-
- @JsonProperty public KafkaSupervisorIOConfig getIoConfig() {
- return ioConfig;
- }
-
- @JsonProperty public Map<String, Object> getContext() {
- return context;
- }
-
- @Override public String toString() {
- return "KafkaSupervisorSpec{"
- + "dataSchema="
- + dataSchema
- + ", tuningConfig="
- + tuningConfig
- + ", ioConfig="
- + ioConfig
- + '}';
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- KafkaSupervisorSpec that = (KafkaSupervisorSpec) o;
-
- if (dataSchema != null ? !dataSchema.equals(that.dataSchema) : that.dataSchema != null) {
- return false;
- }
- if (tuningConfig != null ? !tuningConfig.equals(that.tuningConfig) : that.tuningConfig != null) {
- return false;
- }
- if (ioConfig != null ? !ioConfig.equals(that.ioConfig) : that.ioConfig != null) {
- return false;
- }
- return context != null ? context.equals(that.context) : that.context == null;
- }
-
- @Override public int hashCode() {
- int result = dataSchema != null ? dataSchema.hashCode() : 0;
- result = 31 * result + (tuningConfig != null ? tuningConfig.hashCode() : 0);
- result = 31 * result + (ioConfig != null ? ioConfig.hashCode() : 0);
- result = 31 * result + (context != null ? context.hashCode() : 0);
- return result;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java
deleted file mode 100644
index 4e171612b39..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/KafkaSupervisorTuningConfig.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonSubTypes;
-import com.fasterxml.jackson.annotation.JsonTypeInfo;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.indexing.TuningConfigs;
-import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
-import org.joda.time.Duration;
-import org.joda.time.Period;
-
-import javax.annotation.Nullable;
-import java.io.File;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-@SuppressWarnings("unused")
-@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "type")
-@JsonSubTypes({
- @JsonSubTypes.Type(name = "kafka", value = KafkaSupervisorTuningConfig.class)})
-public class KafkaSupervisorTuningConfig
- extends KafkaIndexTaskTuningConfig implements SeekableStreamSupervisorTuningConfig {
- private static final String DEFAULT_OFFSET_FETCH_PERIOD = "PT30S";
-
- private final Integer workerThreads;
- private final Integer chatThreads;
- private final Long chatRetries;
- private final Duration httpTimeout;
- private final Duration shutdownTimeout;
- private final Duration offsetFetchPeriod;
-
- public static KafkaSupervisorTuningConfig defaultConfig() {
- return new KafkaSupervisorTuningConfig(null, null, null, null, null, null, null, null, null, null, null, null, null,
- null, null, null, null, null, null, null, null, null, null, null);
- }
-
- public KafkaSupervisorTuningConfig(
- @JsonProperty("maxRowsInMemory") Integer maxRowsInMemory,
- @JsonProperty("maxBytesInMemory") Long maxBytesInMemory,
- @JsonProperty("maxRowsPerSegment") Integer maxRowsPerSegment, @JsonProperty("maxTotalRows") Long maxTotalRows,
- @JsonProperty("intermediatePersistPeriod") Period intermediatePersistPeriod,
- @JsonProperty("basePersistDirectory") File basePersistDirectory,
- @JsonProperty("maxPendingPersists") Integer maxPendingPersists, @JsonProperty("indexSpec") IndexSpec indexSpec,
- @JsonProperty("indexSpecForIntermediatePersists") @Nullable IndexSpec indexSpecForIntermediatePersists,
- // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
- @JsonProperty("buildV9Directly") Boolean buildV9Directly,
- @JsonProperty("reportParseExceptions") Boolean reportParseExceptions,
- @JsonProperty("handoffConditionTimeout") Long handoffConditionTimeout,
- @JsonProperty("resetOffsetAutomatically") Boolean resetOffsetAutomatically,
- @JsonProperty("segmentWriteOutMediumFactory") @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
- @JsonProperty("workerThreads") Integer workerThreads, @JsonProperty("chatThreads") Integer chatThreads,
- @JsonProperty("chatRetries") Long chatRetries, @JsonProperty("httpTimeout") Period httpTimeout,
- @JsonProperty("shutdownTimeout") Period shutdownTimeout,
- @JsonProperty("offsetFetchPeriod") Period offsetFetchPeriod,
- @JsonProperty("intermediateHandoffPeriod") Period intermediateHandoffPeriod,
- @JsonProperty("logParseExceptions") @Nullable Boolean logParseExceptions,
- @JsonProperty("maxParseExceptions") @Nullable Integer maxParseExceptions,
- @JsonProperty("maxSavedParseExceptions") @Nullable Integer maxSavedParseExceptions) {
- super(maxRowsInMemory, maxBytesInMemory, maxRowsPerSegment, maxTotalRows, intermediatePersistPeriod,
- basePersistDirectory, maxPendingPersists, indexSpec, indexSpecForIntermediatePersists, true,
- reportParseExceptions, handoffConditionTimeout, resetOffsetAutomatically, segmentWriteOutMediumFactory,
- intermediateHandoffPeriod, logParseExceptions, maxParseExceptions, maxSavedParseExceptions);
- this.workerThreads = workerThreads;
- this.chatThreads = chatThreads;
- this.chatRetries = (chatRetries != null ? chatRetries : DEFAULT_CHAT_RETRIES);
- this.httpTimeout = SeekableStreamSupervisorTuningConfig.defaultDuration(httpTimeout, DEFAULT_HTTP_TIMEOUT);
- this.shutdownTimeout =
- SeekableStreamSupervisorTuningConfig.defaultDuration(shutdownTimeout, DEFAULT_SHUTDOWN_TIMEOUT);
- this.offsetFetchPeriod =
- SeekableStreamSupervisorTuningConfig.defaultDuration(offsetFetchPeriod, DEFAULT_OFFSET_FETCH_PERIOD);
- }
-
- @Override
- @JsonProperty
- public Integer getWorkerThreads() {
- return workerThreads;
- }
-
- @Override
- @JsonProperty
- public Integer getChatThreads() {
- return chatThreads;
- }
-
- @Override
- @JsonProperty
- public Long getChatRetries() {
- return chatRetries;
- }
-
- @Override
- @JsonProperty
- public Duration getHttpTimeout() {
- return httpTimeout;
- }
-
- @Override
- @JsonProperty
- public Duration getShutdownTimeout() {
- return shutdownTimeout;
- }
-
- @Override
- public Duration getRepartitionTransitionDuration() {
- // Stopping tasks early for Kafka ingestion on partition set change is not supported yet,
- // just return a default for now.
- return SeekableStreamSupervisorTuningConfig
- .defaultDuration(null, SeekableStreamSupervisorTuningConfig.DEFAULT_REPARTITION_TRANSITION_DURATION);
- }
-
- @JsonProperty
- public Duration getOffsetFetchPeriod() {
- return offsetFetchPeriod;
- }
-
- @Override
- public String toString() {
- return "KafkaSupervisorTuningConfig{" + "maxRowsInMemory=" + getMaxRowsInMemory() + ", maxRowsPerSegment="
- + getMaxRowsPerSegment() + ", maxTotalRows=" + getMaxTotalRows() + ", maxBytesInMemory=" + TuningConfigs
- .getMaxBytesInMemoryOrDefault(getMaxBytesInMemory()) + ", intermediatePersistPeriod="
- + getIntermediatePersistPeriod() + ", basePersistDirectory=" + getBasePersistDirectory()
- + ", maxPendingPersists=" + getMaxPendingPersists() + ", indexSpec=" + getIndexSpec()
- + ", reportParseExceptions=" + isReportParseExceptions() + ", handoffConditionTimeout="
- + getHandoffConditionTimeout() + ", resetOffsetAutomatically=" + isResetOffsetAutomatically()
- + ", segmentWriteOutMediumFactory=" + getSegmentWriteOutMediumFactory() + ", workerThreads=" + workerThreads
- + ", chatThreads=" + chatThreads + ", chatRetries=" + chatRetries + ", httpTimeout=" + httpTimeout
- + ", shutdownTimeout=" + shutdownTimeout + ", offsetFetchPeriod=" + offsetFetchPeriod
- + ", intermediateHandoffPeriod=" + getIntermediateHandoffPeriod() + ", logParseExceptions="
- + isLogParseExceptions() + ", maxParseExceptions=" + getMaxParseExceptions() + ", maxSavedParseExceptions="
- + getMaxSavedParseExceptions() + '}';
- }
-
- @Override
- public KafkaIndexTaskTuningConfig convertToTaskTuningConfig() {
- return new KafkaIndexTaskTuningConfig(getMaxRowsInMemory(), getMaxBytesInMemory(), getMaxRowsPerSegment(),
- getMaxTotalRows(), getIntermediatePersistPeriod(), getBasePersistDirectory(), getMaxPendingPersists(),
- getIndexSpec(), getIndexSpecForIntermediatePersists(), true, isReportParseExceptions(),
- getHandoffConditionTimeout(), isResetOffsetAutomatically(), getSegmentWriteOutMediumFactory(),
- getIntermediateHandoffPeriod(), isLogParseExceptions(), getMaxParseExceptions(), getMaxSavedParseExceptions());
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java
deleted file mode 100644
index 289f0e8d430..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamIndexTaskTuningConfig.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.apache.druid.indexer.partitions.DynamicPartitionsSpec;
-import org.apache.druid.segment.IndexSpec;
-import org.apache.druid.segment.indexing.RealtimeTuningConfig;
-import org.apache.druid.segment.indexing.TuningConfig;
-import org.apache.druid.segment.realtime.appenderator.AppenderatorConfig;
-import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
-import org.joda.time.Period;
-
-import javax.annotation.Nullable;
-import java.io.File;
-import java.util.Objects;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public abstract class SeekableStreamIndexTaskTuningConfig implements TuningConfig, AppenderatorConfig {
- private static final boolean DEFAULT_RESET_OFFSET_AUTOMATICALLY = false;
- private static final boolean DEFAULT_SKIP_SEQUENCE_NUMBER_AVAILABILITY_CHECK = false;
-
- private final int maxRowsInMemory;
- private final long maxBytesInMemory;
- private final DynamicPartitionsSpec partitionsSpec;
- private final Period intermediatePersistPeriod;
- private final File basePersistDirectory;
- @Deprecated
- private final int maxPendingPersists;
- private final IndexSpec indexSpec;
- private final IndexSpec indexSpecForIntermediatePersists;
- private final boolean reportParseExceptions;
- private final long handoffConditionTimeout;
- private final boolean resetOffsetAutomatically;
- @Nullable
- private final SegmentWriteOutMediumFactory segmentWriteOutMediumFactory;
- private final Period intermediateHandoffPeriod;
- private final boolean skipSequenceNumberAvailabilityCheck;
-
- private final boolean logParseExceptions;
- private final int maxParseExceptions;
- private final int maxSavedParseExceptions;
-
- public SeekableStreamIndexTaskTuningConfig(
- @Nullable Integer maxRowsInMemory,
- @Nullable Long maxBytesInMemory,
- @Nullable Integer maxRowsPerSegment,
- @Nullable Long maxTotalRows,
- @Nullable Period intermediatePersistPeriod,
- @Nullable File basePersistDirectory,
- @Nullable Integer maxPendingPersists,
- @Nullable IndexSpec indexSpec,
- @Nullable IndexSpec indexSpecForIntermediatePersists,
- // This parameter is left for compatibility when reading existing configs, to be removed in Druid 0.12.
- @Deprecated @JsonProperty("buildV9Directly") @Nullable Boolean buildV9Directly,
- @Deprecated @Nullable Boolean reportParseExceptions,
- @Nullable Long handoffConditionTimeout,
- @Nullable Boolean resetOffsetAutomatically,
- Boolean skipSequenceNumberAvailabilityCheck,
- @Nullable SegmentWriteOutMediumFactory segmentWriteOutMediumFactory,
- @Nullable Period intermediateHandoffPeriod,
- @Nullable Boolean logParseExceptions,
- @Nullable Integer maxParseExceptions,
- @Nullable Integer maxSavedParseExceptions
- ) {
- // Cannot be a static because default basePersistDirectory is unique per-instance
- final RealtimeTuningConfig defaults = RealtimeTuningConfig.makeDefaultTuningConfig(basePersistDirectory);
-
- this.maxRowsInMemory = maxRowsInMemory == null ? defaults.getMaxRowsInMemory() : maxRowsInMemory;
- this.partitionsSpec = new DynamicPartitionsSpec(maxRowsPerSegment, maxTotalRows);
- // initializing this to 0, it will be lazily initialized to a value
- // @see server.src.main.java.org.apache.druid.segment.indexing.TuningConfigs#getMaxBytesInMemoryOrDefault(long)
- this.maxBytesInMemory = maxBytesInMemory == null ? 0 : maxBytesInMemory;
- this.intermediatePersistPeriod = intermediatePersistPeriod == null
- ? defaults.getIntermediatePersistPeriod()
- : intermediatePersistPeriod;
- this.basePersistDirectory = defaults.getBasePersistDirectory();
- this.maxPendingPersists = maxPendingPersists == null ? 0 : maxPendingPersists;
- this.indexSpec = indexSpec == null ? defaults.getIndexSpec() : indexSpec;
- this.indexSpecForIntermediatePersists = indexSpecForIntermediatePersists == null ?
- this.indexSpec : indexSpecForIntermediatePersists;
- this.reportParseExceptions = reportParseExceptions == null
- ? defaults.isReportParseExceptions()
- : reportParseExceptions;
- this.handoffConditionTimeout = handoffConditionTimeout == null
- ? defaults.getHandoffConditionTimeout()
- : handoffConditionTimeout;
- this.resetOffsetAutomatically = resetOffsetAutomatically == null
- ? DEFAULT_RESET_OFFSET_AUTOMATICALLY
- : resetOffsetAutomatically;
- this.segmentWriteOutMediumFactory = segmentWriteOutMediumFactory;
- this.intermediateHandoffPeriod = intermediateHandoffPeriod == null
- ? new Period().withDays(Integer.MAX_VALUE)
- : intermediateHandoffPeriod;
- this.skipSequenceNumberAvailabilityCheck = skipSequenceNumberAvailabilityCheck == null
- ? DEFAULT_SKIP_SEQUENCE_NUMBER_AVAILABILITY_CHECK
- : skipSequenceNumberAvailabilityCheck;
-
- if (this.reportParseExceptions) {
- this.maxParseExceptions = 0;
- this.maxSavedParseExceptions = maxSavedParseExceptions == null ? 0 : Math.min(1, maxSavedParseExceptions);
- } else {
- this.maxParseExceptions = maxParseExceptions == null
- ? TuningConfig.DEFAULT_MAX_PARSE_EXCEPTIONS
- : maxParseExceptions;
- this.maxSavedParseExceptions = maxSavedParseExceptions == null
- ? TuningConfig.DEFAULT_MAX_SAVED_PARSE_EXCEPTIONS
- : maxSavedParseExceptions;
- }
- this.logParseExceptions = logParseExceptions == null
- ? TuningConfig.DEFAULT_LOG_PARSE_EXCEPTIONS
- : logParseExceptions;
- }
-
- @Override
- @JsonProperty
- public int getMaxRowsInMemory() {
- return maxRowsInMemory;
- }
-
- @Override
- @JsonProperty
- public long getMaxBytesInMemory() {
- return maxBytesInMemory;
- }
-
- @Override
- @JsonProperty
- public Integer getMaxRowsPerSegment() {
- return partitionsSpec.getMaxRowsPerSegment();
- }
-
- @JsonProperty
- @Override
- @Nullable
- public Long getMaxTotalRows() {
- return partitionsSpec.getMaxTotalRows();
- }
-
- @Override
- public DynamicPartitionsSpec getPartitionsSpec() {
- return partitionsSpec;
- }
-
- @Override
- @JsonProperty
- public Period getIntermediatePersistPeriod() {
- return intermediatePersistPeriod;
- }
-
- @Override
- @JsonProperty
- public File getBasePersistDirectory() {
- return basePersistDirectory;
- }
-
- @Override
- @JsonProperty
- @Deprecated
- public int getMaxPendingPersists() {
- return maxPendingPersists;
- }
-
- @Override
- @JsonProperty
- public IndexSpec getIndexSpec() {
- return indexSpec;
- }
-
- @JsonProperty
- @Override
- public IndexSpec getIndexSpecForIntermediatePersists() {
- return indexSpecForIntermediatePersists;
- }
-
- /**
- * Always returns true, doesn't affect the version being built.
- */
- @Deprecated
- @JsonProperty
- public boolean getBuildV9Directly() {
- return true;
- }
-
- @Override
- @JsonProperty
- public boolean isReportParseExceptions() {
- return reportParseExceptions;
- }
-
- @JsonProperty
- public long getHandoffConditionTimeout() {
- return handoffConditionTimeout;
- }
-
- @JsonProperty
- public boolean isResetOffsetAutomatically() {
- return resetOffsetAutomatically;
- }
-
- @Override
- @JsonProperty
- @Nullable
- public SegmentWriteOutMediumFactory getSegmentWriteOutMediumFactory() {
- return segmentWriteOutMediumFactory;
- }
-
- @JsonProperty
- public Period getIntermediateHandoffPeriod() {
- return intermediateHandoffPeriod;
- }
-
- @JsonProperty
- public boolean isLogParseExceptions() {
- return logParseExceptions;
- }
-
- @JsonProperty
- public int getMaxParseExceptions() {
- return maxParseExceptions;
- }
-
- @JsonProperty
- public int getMaxSavedParseExceptions() {
- return maxSavedParseExceptions;
- }
-
- @JsonProperty
- public boolean isSkipSequenceNumberAvailabilityCheck() {
- return skipSequenceNumberAvailabilityCheck;
- }
-
- @Override
- public abstract SeekableStreamIndexTaskTuningConfig withBasePersistDirectory(File dir);
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- SeekableStreamIndexTaskTuningConfig that = (SeekableStreamIndexTaskTuningConfig) o;
- return maxRowsInMemory == that.maxRowsInMemory &&
- maxBytesInMemory == that.maxBytesInMemory &&
- maxPendingPersists == that.maxPendingPersists &&
- reportParseExceptions == that.reportParseExceptions &&
- handoffConditionTimeout == that.handoffConditionTimeout &&
- resetOffsetAutomatically == that.resetOffsetAutomatically &&
- skipSequenceNumberAvailabilityCheck == that.skipSequenceNumberAvailabilityCheck &&
- logParseExceptions == that.logParseExceptions &&
- maxParseExceptions == that.maxParseExceptions &&
- maxSavedParseExceptions == that.maxSavedParseExceptions &&
- Objects.equals(partitionsSpec, that.partitionsSpec) &&
- Objects.equals(intermediatePersistPeriod, that.intermediatePersistPeriod) &&
- Objects.equals(basePersistDirectory, that.basePersistDirectory) &&
- Objects.equals(indexSpec, that.indexSpec) &&
- Objects.equals(indexSpecForIntermediatePersists, that.indexSpecForIntermediatePersists) &&
- Objects.equals(segmentWriteOutMediumFactory, that.segmentWriteOutMediumFactory) &&
- Objects.equals(intermediateHandoffPeriod, that.intermediateHandoffPeriod);
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(
- maxRowsInMemory,
- maxBytesInMemory,
- partitionsSpec,
- intermediatePersistPeriod,
- basePersistDirectory,
- maxPendingPersists,
- indexSpec,
- indexSpecForIntermediatePersists,
- reportParseExceptions,
- handoffConditionTimeout,
- resetOffsetAutomatically,
- segmentWriteOutMediumFactory,
- intermediateHandoffPeriod,
- skipSequenceNumberAvailabilityCheck,
- logParseExceptions,
- maxParseExceptions,
- maxSavedParseExceptions
- );
- }
-
- @Override
- public abstract String toString();
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java
deleted file mode 100644
index ec71518f46c..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/SeekableStreamSupervisorTuningConfig.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import org.joda.time.Duration;
-import org.joda.time.Period;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-public interface SeekableStreamSupervisorTuningConfig {
-
- int DEFAULT_CHAT_RETRIES = 8;
- String DEFAULT_HTTP_TIMEOUT = "PT10S";
- String DEFAULT_SHUTDOWN_TIMEOUT = "PT80S";
- String DEFAULT_REPARTITION_TRANSITION_DURATION = "PT2M";
-
- static Duration defaultDuration(final Period period, final String theDefault) {
- return (period == null ? new Period(theDefault) : period).toStandardDuration();
- }
-
- @JsonProperty
- Integer getWorkerThreads();
-
- @JsonProperty
- Integer getChatThreads();
-
- @JsonProperty
- Long getChatRetries();
-
- @JsonProperty
- Duration getHttpTimeout();
-
- @JsonProperty
- Duration getShutdownTimeout();
-
- @JsonProperty
- Duration getRepartitionTransitionDuration();
-
- SeekableStreamIndexTaskTuningConfig convertToTaskTuningConfig();
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java
deleted file mode 100644
index f35db6d01cb..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/TaskReportData.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.json;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.annotation.JsonProperty;
-
-import org.joda.time.DateTime;
-
-import java.util.Map;
-
-import javax.annotation.Nullable;
-
-/**
- * This class is copied from druid source code
- * in order to avoid adding additional dependencies on druid-indexing-service.
- */
-@SuppressWarnings("unused") public class TaskReportData {
- /**
- * Task type used by serializer but does not have any functionality as far i can tell.
- */
- @SuppressWarnings("unused") public enum TaskType {
- ACTIVE, PUBLISHING, UNKNOWN
- }
-
- private final String id;
- private final Map<Integer, Long> startingOffsets;
- private final DateTime startTime;
- private final Long remainingSeconds;
- private final TaskType type;
- private final Map<Integer, Long> currentOffsets;
- private final Map<Integer, Long> lag;
-
- public TaskReportData(String id,
- @Nullable Map<Integer, Long> startingOffsets,
- @Nullable Map<Integer, Long> currentOffsets,
- DateTime startTime,
- Long remainingSeconds,
- TaskType type,
- @Nullable Map<Integer, Long> lag) {
- this.id = id;
- this.startingOffsets = startingOffsets;
- this.currentOffsets = currentOffsets;
- this.startTime = startTime;
- this.remainingSeconds = remainingSeconds;
- this.type = type;
- this.lag = lag;
- }
-
- @JsonProperty public String getId() {
- return id;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map<Integer, Long> getStartingOffsets() {
- return startingOffsets;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map<Integer, Long> getCurrentOffsets() {
- return currentOffsets;
- }
-
- @JsonProperty public DateTime getStartTime() {
- return startTime;
- }
-
- @JsonProperty public Long getRemainingSeconds() {
- return remainingSeconds;
- }
-
- @JsonProperty public TaskType getType() {
- return type;
- }
-
- @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) public Map<Integer, Long> getLag() {
- return lag;
- }
-
- @Override public String toString() {
- return "{" + "id='" + id + '\'' + (startingOffsets != null ? ", startingOffsets=" + startingOffsets : "") + (
- currentOffsets != null ?
- ", currentOffsets=" + currentOffsets :
- "") + ", startTime=" + startTime + ", remainingSeconds=" + remainingSeconds + (lag != null ?
- ", lag=" + lag :
- "") + '}';
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java
deleted file mode 100644
index dc53fa7de70..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/json/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * package info file.
- */
-
-package org.apache.hadoop.hive.druid.json;
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java
deleted file mode 100644
index 19acf64bb93..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package info file.
- */
-
-package org.apache.hadoop.hive.druid;
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
deleted file mode 100644
index 12603c10ec7..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/DruidKerberosUtil.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.security;
-
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
-import org.apache.hadoop.security.authentication.util.KerberosUtil;
-import org.ietf.jgss.GSSContext;
-import org.ietf.jgss.GSSException;
-import org.ietf.jgss.GSSManager;
-import org.ietf.jgss.GSSName;
-import org.ietf.jgss.Oid;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.CookieStore;
-import java.net.HttpCookie;
-import java.net.URI;
-import java.util.Base64;
-import java.util.List;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Utils class for Druid Kerberos stuff.
- */
-public final class DruidKerberosUtil {
- protected static final Logger LOG = LoggerFactory.getLogger(DruidKerberosUtil.class);
- private static final ReentrantLock KERBEROS_LOCK = new ReentrantLock(true);
-
- private DruidKerberosUtil() {
- }
-
- /**
- * This method always needs to be called within a doAs block so that the client's TGT credentials
- * can be read from the Subject.
- *
- * @return Kerberos Challenge String.
- *
- * @throws AuthenticationException on authentication errors.
- */
-
- static String kerberosChallenge(String server) throws AuthenticationException {
- KERBEROS_LOCK.lock();
- try {
- // This Oid for Kerberos GSS-API mechanism.
- Oid mechOid = KerberosUtil.getOidInstance("GSS_KRB5_MECH_OID");
- GSSManager manager = GSSManager.getInstance();
- // GSS name for server
- GSSName serverName = manager.createName("HTTP@" + server, GSSName.NT_HOSTBASED_SERVICE);
- // Create a GSSContext for authentication with the service.
- // We're passing client credentials as null since we want them to be read from the Subject.
- GSSContext
- gssContext =
- manager.createContext(serverName.canonicalize(mechOid), mechOid, null, GSSContext.DEFAULT_LIFETIME);
- gssContext.requestMutualAuth(true);
- gssContext.requestCredDeleg(true);
- // Establish context
- byte[] inToken = new byte[0];
- byte[] outToken = gssContext.initSecContext(inToken, 0, inToken.length);
- gssContext.dispose();
- // Base64 encoded and stringified token for server
- LOG.debug("Got valid challenge for host {}", serverName);
- return Base64.getEncoder().encodeToString(outToken);
- } catch (GSSException | IllegalAccessException | NoSuchFieldException | ClassNotFoundException e) {
- throw new AuthenticationException(e);
- } finally {
- KERBEROS_LOCK.unlock();
- }
- }
-
- static HttpCookie getAuthCookie(CookieStore cookieStore, URI uri) {
- if (cookieStore == null) {
- return null;
- }
- boolean isSSL = uri.getScheme().equals("https");
- List<HttpCookie> cookies = cookieStore.getCookies();
-
- for (HttpCookie c : cookies) {
- // If this is a secured cookie and the current connection is non-secured,
- // then, skip this cookie. We need to skip this cookie because, the cookie
- // replay will not be transmitted to the server.
- if (c.getSecure() && !isSSL) {
- continue;
- }
- if (c.getName().equals(AuthenticatedURL.AUTH_COOKIE)) {
- return c;
- }
- }
- return null;
- }
-
- static void removeAuthCookie(CookieStore cookieStore, URI uri) {
- HttpCookie authCookie = getAuthCookie(cookieStore, uri);
- if (authCookie != null) {
- cookieStore.remove(uri, authCookie);
- }
- }
-
- static boolean needToSendCredentials(CookieStore cookieStore, URI uri) {
- return getAuthCookie(cookieStore, uri) == null;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
deleted file mode 100644
index fdbbfcc7d7f..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/KerberosHttpClient.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.security;
-
-import com.google.common.base.Throwables;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.SettableFuture;
-import org.apache.druid.java.util.http.client.AbstractHttpClient;
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.java.util.http.client.response.HttpResponseHandler;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.jboss.netty.handler.codec.http.HttpHeaders;
-import org.joda.time.Duration;
-import org.slf4j.LoggerFactory;
-
-import java.net.CookieManager;
-import java.net.URI;
-import java.security.PrivilegedExceptionAction;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-/**
- * This is a slightly modified version of kerberos module borrowed from druid project
- * Couple of reasons behind the copy/modification instead of mvn dependency.
- * 1/ Need to remove the authentication step since it not required
- * 2/ To avoid some un-needed transitive dependencies that can clash on the classpath like jetty-XX.
- */
-public class KerberosHttpClient extends AbstractHttpClient {
- protected static final org.slf4j.Logger LOG = LoggerFactory.getLogger(KerberosHttpClient.class);
- private final HttpClient delegate;
- private final CookieManager cookieManager;
-
- public KerberosHttpClient(HttpClient delegate) {
- this.delegate = delegate;
- this.cookieManager = new CookieManager();
- }
-
- @Override public <Intermediate, Final> ListenableFuture<Final> go(Request request,
- HttpResponseHandler<Intermediate, Final> httpResponseHandler,
- Duration duration) {
- final SettableFuture<Final> retVal = SettableFuture.create();
- innerGo(request, httpResponseHandler, duration, retVal);
- return retVal;
- }
-
- private <Intermediate, Final> void innerGo(final Request request,
- final HttpResponseHandler<Intermediate, Final> httpResponseHandler,
- final Duration duration,
- final SettableFuture<Final> future) {
- try {
- final String host = request.getUrl().getHost();
- final URI uri = request.getUrl().toURI();
-
- /* Cookies Manager is used to cache cookie returned by service.
- The goal us to avoid doing KDC requests for every request.*/
-
- Map<String, List<String>> cookieMap = cookieManager.get(uri, Collections.emptyMap());
- for (Map.Entry<String, List<String>> entry : cookieMap.entrySet()) {
- request.addHeaderValues(entry.getKey(), entry.getValue());
- }
- final boolean shouldRetryOnUnauthorizedResponse;
-
- if (DruidKerberosUtil.needToSendCredentials(cookieManager.getCookieStore(), uri)) {
- // No Cookies for requested URI, authenticate user and add authentication header
- LOG.debug("No Auth Cookie found for URI{}. Existing Cookies{} Authenticating... ",
- uri,
- cookieManager.getCookieStore().getCookies());
- // Assuming that a valid UGI with kerberos cred is created by HS2 or LLAP
- UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
- currentUser.checkTGTAndReloginFromKeytab();
- LOG.debug("The user credential is {}", currentUser);
- String
- challenge =
- currentUser.doAs((PrivilegedExceptionAction<String>) () -> DruidKerberosUtil.kerberosChallenge(host));
- request.setHeader(HttpHeaders.Names.AUTHORIZATION, "Negotiate " + challenge);
- /* no reason to retry if the challenge ticket is not valid. */
- shouldRetryOnUnauthorizedResponse = false;
- } else {
- /* In this branch we had already a cookie that did expire
- therefore we need to resend a valid Kerberos challenge*/
- LOG.debug("Found Auth Cookie found for URI {} cookie {}",
- uri,
- DruidKerberosUtil.getAuthCookie(cookieManager.getCookieStore(), uri).toString());
- shouldRetryOnUnauthorizedResponse = true;
- }
-
- @SuppressWarnings("unchecked") ListenableFuture<RetryResponseHolder<Final>>
- internalFuture =
- delegate.go(request,
- new RetryIfUnauthorizedResponseHandler<Intermediate, Final>(new ResponseCookieHandler(request.getUrl()
- .toURI(), cookieManager, httpResponseHandler)),
- duration);
-
- RetryResponseHolder<Final> responseHolder = internalFuture.get();
-
- if (shouldRetryOnUnauthorizedResponse && responseHolder.shouldRetry()) {
- LOG.debug("Preparing for Retry boolean {} and result {}, object{} ", true,
- responseHolder.shouldRetry(),
- responseHolder.getObj());
- // remove Auth cookie
- DruidKerberosUtil.removeAuthCookie(cookieManager.getCookieStore(), uri);
- // clear existing cookie
- request.setHeader("Cookie", "");
- innerGo(request.copy(), httpResponseHandler, duration, future);
-
- } else {
- future.set(responseHolder.getObj());
- }
- } catch (Throwable e) {
- throw Throwables.propagate(e);
- }
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java
deleted file mode 100644
index 223000e62a0..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/ResponseCookieHandler.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.security;
-
-import com.google.common.collect.Maps;
-import org.apache.druid.java.util.http.client.response.ClientResponse;
-import org.apache.druid.java.util.http.client.response.HttpResponseHandler;
-import org.jboss.netty.handler.codec.http.HttpChunk;
-import org.jboss.netty.handler.codec.http.HttpHeaders;
-import org.jboss.netty.handler.codec.http.HttpResponse;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.CookieManager;
-import java.net.URI;
-
-/**
- * Class to handle Cookies used to cache the Kerberos Credentials.
- * @param <Intermediate> intermediate response type.
- * @param <Final> final response type.
- */
-public class ResponseCookieHandler<Intermediate, Final> implements HttpResponseHandler<Intermediate, Final> {
- private static final Logger LOG = LoggerFactory.getLogger(ResponseCookieHandler.class);
-
- private final URI uri;
- private final CookieManager manager;
- private final HttpResponseHandler<Intermediate, Final> delegate;
-
- ResponseCookieHandler(URI uri, CookieManager manager, HttpResponseHandler<Intermediate, Final> delegate) {
- this.uri = uri;
- this.manager = manager;
- this.delegate = delegate;
- }
-
- @Override public ClientResponse<Intermediate> handleResponse(HttpResponse httpResponse, TrafficCop trafficCop) {
- try {
- final HttpHeaders headers = httpResponse.headers();
- manager.put(uri, Maps.asMap(headers.names(), headers::getAll));
- return delegate.handleResponse(httpResponse, trafficCop);
- } catch (IOException e) {
- LOG.error("Error while processing Cookies from header", e);
- throw new RuntimeException(e);
- }
- }
-
- @Override public ClientResponse<Intermediate> handleChunk(ClientResponse<Intermediate> clientResponse,
- HttpChunk httpChunk, long chunkNum) {
- return delegate.handleChunk(clientResponse, httpChunk, chunkNum);
- }
-
- @Override public ClientResponse<Final> done(ClientResponse<Intermediate> clientResponse) {
- return delegate.done(clientResponse);
- }
-
- @Override public void exceptionCaught(ClientResponse<Intermediate> clientResponse, Throwable throwable) {
- delegate.exceptionCaught(clientResponse, throwable);
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java
deleted file mode 100644
index d6702f5a1fe..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryIfUnauthorizedResponseHandler.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.security;
-
-import org.apache.druid.java.util.http.client.response.ClientResponse;
-import org.apache.druid.java.util.http.client.response.HttpResponseHandler;
-import org.jboss.netty.handler.codec.http.HttpChunk;
-import org.jboss.netty.handler.codec.http.HttpResponse;
-import org.jboss.netty.handler.codec.http.HttpResponseStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class handling retry on Unauthorized responses.
- *
- * @param <Intermediate> Intermediate response type.
- * @param <Final> final result type.
- */
-public class RetryIfUnauthorizedResponseHandler<Intermediate, Final>
- implements HttpResponseHandler<RetryResponseHolder<Intermediate>, RetryResponseHolder<Final>> {
- protected static final Logger LOG = LoggerFactory.getLogger(RetryIfUnauthorizedResponseHandler.class);
-
- private final HttpResponseHandler<Intermediate, Final> httpResponseHandler;
-
- public RetryIfUnauthorizedResponseHandler(HttpResponseHandler<Intermediate, Final> httpResponseHandler) {
- this.httpResponseHandler = httpResponseHandler;
- }
-
- @Override public ClientResponse<RetryResponseHolder<Intermediate>> handleResponse(HttpResponse httpResponse,
- TrafficCop trafficCop) {
- LOG.debug("UnauthorizedResponseHandler - Got response status {}", httpResponse.getStatus());
- if (httpResponse.getStatus().equals(HttpResponseStatus.UNAUTHORIZED)) {
- // Drain the buffer
- //noinspection ResultOfMethodCallIgnored
- httpResponse.getContent().toString();
- return ClientResponse.unfinished(RetryResponseHolder.retry());
- } else {
- return wrap(httpResponseHandler.handleResponse(httpResponse, trafficCop));
- }
- }
-
- @Override public ClientResponse<RetryResponseHolder<Intermediate>> handleChunk(
- ClientResponse<RetryResponseHolder<Intermediate>> clientResponse,
- HttpChunk httpChunk, long chunkNum) {
- if (clientResponse.getObj().shouldRetry()) {
- // Drain the buffer
- //noinspection ResultOfMethodCallIgnored
- httpChunk.getContent().toString();
- return clientResponse;
- } else {
- return wrap(httpResponseHandler.handleChunk(unwrap(clientResponse), httpChunk, chunkNum));
- }
- }
-
- @Override public ClientResponse<RetryResponseHolder<Final>> done(
- ClientResponse<RetryResponseHolder<Intermediate>> clientResponse) {
- if (clientResponse.getObj().shouldRetry()) {
- return ClientResponse.finished(RetryResponseHolder.retry());
- } else {
- return wrap(httpResponseHandler.done(unwrap(clientResponse)));
- }
- }
-
- @Override public void exceptionCaught(ClientResponse<RetryResponseHolder<Intermediate>> clientResponse,
- Throwable throwable) {
- httpResponseHandler.exceptionCaught(unwrap(clientResponse), throwable);
- }
-
- private <T> ClientResponse<RetryResponseHolder<T>> wrap(ClientResponse<T> response) {
- if (response.isFinished()) {
- return ClientResponse.finished(new RetryResponseHolder<>(false, response.getObj()));
- } else {
- return ClientResponse.unfinished(new RetryResponseHolder<>(false, response.getObj()));
- }
- }
-
- private <T> ClientResponse<T> unwrap(ClientResponse<RetryResponseHolder<T>> response) {
- if (response.isFinished()) {
- return ClientResponse.finished(response.getObj().getObj());
- } else {
- return ClientResponse.unfinished(response.getObj().getObj());
- }
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java
deleted file mode 100644
index 067107e1d3c..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/RetryResponseHolder.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.security;
-
-/**
- * Retry Response Handler class.
- * @param <T> type of the response object.
- */
-public class RetryResponseHolder<T> {
- private final boolean shouldRetry;
- private final T obj;
-
- public RetryResponseHolder(boolean shouldRetry, T obj) {
- this.shouldRetry = shouldRetry;
- this.obj = obj;
- }
-
- public static <T> RetryResponseHolder<T> retry() {
- return new RetryResponseHolder<>(true, null);
- }
-
- public boolean shouldRetry() {
- return shouldRetry;
- }
-
- public T getObj() {
- return obj;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java
deleted file mode 100644
index 5d8201e73ac..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/security/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package info file.
- */
-
-package org.apache.hadoop.hive.druid.security;
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java
deleted file mode 100644
index 4def3864c72..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidGroupByQueryRecordReader.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JavaType;
-import org.apache.druid.data.input.MapBasedRow;
-import org.apache.druid.data.input.Row;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.io.NullWritable;
-
-import java.io.IOException;
-import java.util.Map;
-
-/**
- * Record reader for results for Druid GroupByQuery.
- */
-public class DruidGroupByQueryRecordReader extends DruidQueryRecordReader<Row> {
- private static final TypeReference<Row> TYPE_REFERENCE = new TypeReference<Row>() {
- };
-
- private MapBasedRow currentRow;
- private Map<String, Object> currentEvent;
-
- @Override protected JavaType getResultTypeDef() {
- return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE);
- }
-
- @Override public boolean nextKeyValue() {
- // Results
-
- if (getQueryResultsIterator().hasNext()) {
- final Row row = getQueryResultsIterator().next();
- // currently druid supports only MapBasedRow as Jackson SerDe so it should safe to cast without check
- currentRow = (MapBasedRow) row;
- currentEvent = currentRow.getEvent();
- return true;
- }
- return false;
- }
-
- @Override public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException {
- // Create new value
- DruidWritable value = new DruidWritable(false);
- // 1) The timestamp column
- value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN,
- currentRow.getTimestamp() == null ? null : currentRow.getTimestamp().getMillis()
- );
- // 2) The dimension columns
- value.getValue().putAll(currentEvent);
- return value;
- }
-
- @Override public boolean next(NullWritable key, DruidWritable value) {
- if (nextKeyValue()) {
- // Update value
- value.getValue().clear();
- // 1) The timestamp column
- value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN,
- currentRow.getTimestamp() == null ? null : currentRow.getTimestamp().getMillis()
- );
- // 2) The dimension columns
- value.getValue().putAll(currentEvent);
- return true;
- }
- return false;
- }
-
- @Override public float getProgress() throws IOException {
- return getQueryResultsIterator().hasNext() ? 0 : 1;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java
deleted file mode 100644
index 19379e1724d..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidQueryRecordReader.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.fasterxml.jackson.core.ObjectCodec;
-import com.fasterxml.jackson.databind.JavaType;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Throwables;
-import org.apache.druid.java.util.common.IAE;
-import org.apache.druid.java.util.common.RE;
-import org.apache.druid.java.util.common.guava.CloseQuietly;
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.java.util.http.client.Request;
-import org.apache.druid.java.util.http.client.response.InputStreamResponseHandler;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.QueryInterruptedException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.druid.DruidStorageHandler;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.io.HiveDruidSplit;
-import org.apache.hadoop.hive.ql.exec.SerializationUtilities;
-import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.TableScanDesc;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.InputSplit;
-import org.apache.hadoop.mapreduce.RecordReader;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
-import org.apache.parquet.Strings;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.Iterator;
-import java.util.Objects;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-
-/**
- * Base record reader for given a Druid query. This class contains the logic to
- * send the query to the broker and retrieve the results. The transformation to
- * emit records needs to be done by the classes that extend the reader.
- * <p>
- * The key for each record will be a NullWritable, while the value will be a
- * DruidWritable containing the timestamp as well as all values resulting from
- * the query.
- */
-public abstract class DruidQueryRecordReader<R extends Comparable<R>> extends RecordReader<NullWritable, DruidWritable>
- implements org.apache.hadoop.mapred.RecordReader<NullWritable, DruidWritable> {
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidQueryRecordReader.class);
- private final Object initLock = new Object();
- // Smile mapper is used to read query results that are serialized as binary instead of json
- private ObjectMapper smileMapper;
- private Configuration conf;
- private String[] locations;
- private HttpClient httpClient;
- /**
- * Query that Druid executes.
- */
- protected Query query;
-
- /**
- * Query results as a streaming iterator.
- */
- private volatile JsonParserIterator<R> queryResultsIterator = null;
-
- public JsonParserIterator<R> getQueryResultsIterator() {
- if (this.queryResultsIterator == null) {
- synchronized (initLock) {
- if (this.queryResultsIterator == null) {
- this.queryResultsIterator = createQueryResultsIterator();
- }
- }
- }
- return this.queryResultsIterator;
- }
-
- public JsonParserIterator<R> createQueryResultsIterator() {
- JsonParserIterator<R> iterator = null;
- String filterExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
- if (filterExprSerialized != null) {
- ExprNodeGenericFuncDesc filterExpr = SerializationUtilities.deserializeExpression(filterExprSerialized);
- query = DruidStorageHandlerUtils.addDynamicFilters(query, filterExpr, conf, true);
- }
-
- // Result type definition used to read the rows, this is query dependent.
- JavaType resultsType = getResultTypeDef();
- boolean initialized = false;
- int currentLocationIndex = 0;
- Exception ex = null;
- while (!initialized && currentLocationIndex < locations.length) {
- String address = locations[currentLocationIndex++];
- if (Strings.isNullOrEmpty(address)) {
- throw new RE("can not fetch results from empty or null host value");
- }
- // Execute query
- LOG.debug("Retrieving data from druid location[{}] using query:[{}] ", address, query);
- try {
- Request request = DruidStorageHandlerUtils.createSmileRequest(address, query);
- Future<InputStream> inputStreamFuture = httpClient.go(request, new InputStreamResponseHandler());
- //noinspection unchecked
- iterator =
- new JsonParserIterator(smileMapper, resultsType, inputStreamFuture, request.getUrl().toString(), query);
- iterator.init();
- initialized = true;
- } catch (Exception e) {
- if (iterator != null) {
- // We got exception while querying results from this host.
- CloseQuietly.close(iterator);
- }
- LOG.error("Failure getting results for query[{}] from host[{}] because of [{}]",
- query, address, e.getMessage());
- if (ex == null) {
- ex = e;
- } else {
- ex.addSuppressed(e);
- }
- }
- }
-
- if (!initialized) {
- throw new RE(ex, "Failure getting results for query[%s] from locations[%s] because of [%s]", query, locations,
- Objects.requireNonNull(ex).getMessage());
- }
- return iterator;
- }
-
- @Override public void initialize(InputSplit split, TaskAttemptContext context) throws IOException {
- initialize(split, context.getConfiguration());
- }
-
- public void initialize(InputSplit split, ObjectMapper mapper, ObjectMapper smileMapper, HttpClient httpClient,
- Configuration conf) throws IOException {
- this.conf = conf;
- HiveDruidSplit hiveDruidSplit = (HiveDruidSplit) split;
- Preconditions.checkNotNull(hiveDruidSplit, "input split is null ???");
- Preconditions.checkNotNull(httpClient, "need Http Client can not be null");
- // Smile mapper is used to read query results that are serialized as binary instead of json
- this.smileMapper = Preconditions.checkNotNull(smileMapper, "Smile Mapper can not be null");
- // Create query
- this.query = mapper.readValue(Preconditions.checkNotNull(hiveDruidSplit.getDruidQuery()), Query.class);
- Preconditions.checkNotNull(query);
-
- this.locations = hiveDruidSplit.getLocations();
- this.httpClient = httpClient;
- }
-
- public void initialize(InputSplit split, Configuration conf) throws IOException {
- initialize(split,
- DruidStorageHandlerUtils.JSON_MAPPER,
- DruidStorageHandlerUtils.SMILE_MAPPER, DruidStorageHandler.getHttpClient(), conf);
- }
-
- protected abstract JavaType getResultTypeDef();
-
- @Override public NullWritable createKey() {
- return NullWritable.get();
- }
-
- @Override public DruidWritable createValue() {
- return new DruidWritable(false);
- }
-
- @Override public abstract boolean next(NullWritable key, DruidWritable value) throws IOException;
-
- @Override public long getPos() {
- // HiveContextAwareRecordReader uses this position to track the block position and check
- // whether to skip header and footer. return -1 to since we need not skip any header and
- // footer rows for druid.
- return -1;
- }
-
- @Override public abstract boolean nextKeyValue() throws IOException;
-
- @Override public abstract NullWritable getCurrentKey() throws IOException, InterruptedException;
-
- @Override public abstract DruidWritable getCurrentValue() throws IOException, InterruptedException;
-
- @Override public abstract float getProgress() throws IOException;
-
- @Override public void close() {
- if (queryResultsIterator != null) {
- CloseQuietly.close(queryResultsIterator);
- }
- }
-
- /**
- * This is a helper wrapper class used to create an iterator of druid rows out of InputStream.
- * The type of the rows is defined by
- * org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader.JsonParserIterator#typeRef
- *
- * @param <R> druid Row type returned as result
- */
- @SuppressWarnings("TypeParameterHidesVisibleType") protected class JsonParserIterator<R extends Comparable<R>>
- implements Iterator<R>, Closeable {
- private JsonParser jp;
- private ObjectCodec objectCodec;
- private final ObjectMapper mapper;
- private final JavaType typeRef;
- private final Future<InputStream> future;
- private final Query query;
- private final String url;
-
- /**
- * @param mapper mapper used to deserialize the stream of data (we use smile factory)
- * @param typeRef Type definition of the results objects
- * @param future Future holding the input stream (the input stream is not owned but it will be closed
- * when org.apache.hadoop.hive.druid.serde.DruidQueryRecordReader.JsonParserIterator#close() is
- * called
- * or reach the end of the steam)
- * @param url URL used to fetch the data, used mostly as message with exception stack to identify the faulty
- * stream,
- * thus this can be empty string.
- * @param query Query used to fetch the data, used mostly as message with exception stack, thus can be empty
- * string.
- */
- JsonParserIterator(ObjectMapper mapper, JavaType typeRef, Future<InputStream> future, String url, Query query) {
- this.typeRef = typeRef;
- this.future = future;
- this.url = url;
- this.query = query;
- this.mapper = mapper;
- jp = null;
- }
-
- @Override public boolean hasNext() {
- if (jp.isClosed()) {
- return false;
- }
- if (jp.getCurrentToken() == JsonToken.END_ARRAY) {
- CloseQuietly.close(jp);
- return false;
- }
-
- return true;
- }
-
- @Override public R next() {
- try {
- final R retVal = objectCodec.readValue(jp, typeRef);
- jp.nextToken();
- return retVal;
- } catch (IOException e) {
- throw Throwables.propagate(e);
- }
- }
-
- @Override public void remove() {
- throw new UnsupportedOperationException();
- }
-
- private void init() {
- if (jp == null) {
- try {
- InputStream is = future.get();
- if (is == null) {
- throw new IOException(String.format("query[%s] url[%s] timed out", query, url));
- } else {
- jp = mapper.getFactory().createParser(is).configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, true);
- }
- final JsonToken nextToken = jp.nextToken();
- if (nextToken == JsonToken.START_OBJECT) {
- QueryInterruptedException cause = jp.getCodec().readValue(jp, QueryInterruptedException.class);
- throw new QueryInterruptedException(cause);
- } else if (nextToken != JsonToken.START_ARRAY) {
- throw new IAE("Next token wasn't a START_ARRAY, was[%s] from url [%s]", jp.getCurrentToken(), url);
- } else {
- jp.nextToken();
- objectCodec = jp.getCodec();
- }
-
- } catch (IOException | InterruptedException | ExecutionException e) {
- throw new RE(e, "Failure getting results for query[%s] url[%s] because of [%s]", query, url, e.getMessage());
- }
- }
- }
-
- @Override public void close() throws IOException {
- CloseQuietly.close(jp);
- }
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java
deleted file mode 100644
index 9afb76a12f4..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidScanQueryRecordReader.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import org.apache.druid.query.scan.ScanResultValue;
-
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.io.NullWritable;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JavaType;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * Record reader for results for Druid ScanQuery.
- */
-public class DruidScanQueryRecordReader extends DruidQueryRecordReader<ScanResultValue> {
-
- private static final TypeReference<ScanResultValue> TYPE_REFERENCE = new TypeReference<ScanResultValue>() {
- };
-
- private Iterator<List<Object>> compactedValues = Collections.emptyIterator();
-
- @Override protected JavaType getResultTypeDef() {
- return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE);
- }
-
- @Override public boolean nextKeyValue() throws IOException {
- if (compactedValues.hasNext()) {
- return true;
- }
- if (getQueryResultsIterator().hasNext()) {
- ScanResultValue current = getQueryResultsIterator().next();
- //noinspection unchecked
- compactedValues = ((List<List<Object>>) current.getEvents()).iterator();
- return nextKeyValue();
- }
- return false;
- }
-
- @Override public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override public DruidWritable createValue() {
- return new DruidWritable(true);
- }
-
- @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException {
- // Create new value
- DruidWritable value = new DruidWritable(true);
- value.setCompactedValue(compactedValues.next());
- return value;
- }
-
- @Override public boolean next(NullWritable key, DruidWritable value) throws IOException {
- if (nextKeyValue()) {
- // Update value
- value.setCompactedValue(compactedValues.next());
- return true;
- }
- return false;
- }
-
- @Override public float getProgress() {
- return getQueryResultsIterator().hasNext() || compactedValues.hasNext() ? 0 : 1;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
deleted file mode 100644
index 25a35ea47d5..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDe.java
+++ /dev/null
@@ -1,497 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import org.apache.druid.query.Druids;
-import org.apache.druid.query.Druids.SegmentMetadataQueryBuilder;
-import org.apache.druid.query.metadata.metadata.ColumnAnalysis;
-import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
-import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.type.Date;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.Timestamp;
-import org.apache.hadoop.hive.common.type.TimestampTZ;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.DruidStorageHandler;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.SerDeSpec;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.StringUtils;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.time.Instant;
-import java.time.ZonedDateTime;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.stream.Collectors;
-
-import static org.joda.time.format.ISODateTimeFormat.dateOptionalTimeParser;
-
-/**
- * DruidSerDe that is used to deserialize objects from a Druid data source.
- */
-@SerDeSpec(schemaProps = { Constants.DRUID_DATA_SOURCE })
-public class DruidSerDe extends AbstractSerDe {
-
- private String[] columns;
- private PrimitiveTypeInfo[] types;
- private ObjectInspector inspector;
- private TimestampLocalTZTypeInfo tsTZTypeInfo;
-
- @Override
- public void initialize(Configuration configuration, Properties tableProperties, Properties partitionProperties)
- throws SerDeException {
- super.initialize(configuration, tableProperties, partitionProperties);
-
- tsTZTypeInfo = new TimestampLocalTZTypeInfo(configuration.get(HiveConf.ConfVars.HIVE_LOCAL_TIME_ZONE.varname));
- // Druid query
- final String druidQuery = properties.getProperty(Constants.DRUID_QUERY_JSON, null);
- if (druidQuery != null && !druidQuery.isEmpty()) {
- initFromDruidQueryPlan(properties, druidQuery);
- } else {
- // No query. Either it is a CTAS, or we need to create a Druid meta data Query
- if (!org.apache.commons.lang3.StringUtils.isEmpty(properties.getProperty(serdeConstants.LIST_COLUMNS))
- && !org.apache.commons.lang3.StringUtils.isEmpty(properties.getProperty(serdeConstants.LIST_COLUMN_TYPES))) {
- // CASE CTAS statement
- initFromProperties(properties);
- } else {
- // Segment Metadata query that retrieves all columns present in
- // the data source (dimensions and metrics).
- initFromMetaDataQuery(configuration, properties);
- }
- }
- if (log.isDebugEnabled()) {
- log.debug("DruidSerDe initialized with\n"
- + "\t columns: "
- + Arrays.toString(columns)
- + "\n\t types: "
- + Arrays.toString(types));
- }
- }
-
- private void initFromMetaDataQuery(final Configuration configuration, final Properties properties)
- throws SerDeException {
- final List<String> columnNames = new ArrayList<>();
- final List<PrimitiveTypeInfo> columnTypes = new ArrayList<>();
- final List<ObjectInspector> inspectors = new ArrayList<>();
-
- String dataSource = properties.getProperty(Constants.DRUID_DATA_SOURCE);
- if (dataSource == null) {
- throw new SerDeException("Druid data source not specified; use "
- + Constants.DRUID_DATA_SOURCE
- + " in table properties");
- }
- SegmentMetadataQueryBuilder builder = new Druids.SegmentMetadataQueryBuilder();
- builder.dataSource(dataSource);
- builder.merge(true);
- builder.analysisTypes();
- SegmentMetadataQuery query = builder.build();
-
- // Execute query in Druid
- String address = HiveConf.getVar(configuration, HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS);
- if (org.apache.commons.lang3.StringUtils.isEmpty(address)) {
- throw new SerDeException("Druid broker address not specified in configuration");
- }
- // Infer schema
- SegmentAnalysis schemaInfo;
- try {
- schemaInfo = submitMetadataRequest(address, query);
- } catch (IOException e) {
- throw new SerDeException(e);
- }
- for (Entry<String, ColumnAnalysis> columnInfo : schemaInfo.getColumns().entrySet()) {
- if (columnInfo.getKey().equals(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- // Special handling for timestamp column
- columnNames.add(columnInfo.getKey()); // field name
- PrimitiveTypeInfo type = tsTZTypeInfo; // field type
- columnTypes.add(type);
- inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type));
- continue;
- }
- columnNames.add(columnInfo.getKey()); // field name
- PrimitiveTypeInfo type = DruidSerDeUtils.convertDruidToHiveType(columnInfo.getValue().getType()); // field type
- columnTypes.add(type instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : type);
- inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(type));
- }
- columns = columnNames.toArray(new String[0]);
- types = columnTypes.toArray(new PrimitiveTypeInfo[0]);
- inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
- }
-
- private void initFromProperties(final Properties properties) throws SerDeException {
-
- final List<String> columnNames = new ArrayList<>(Utilities.getColumnNames(properties));
- if (!columnNames.contains(DruidConstants.DEFAULT_TIMESTAMP_COLUMN)) {
- throw new SerDeException("Timestamp column (' "
- + DruidConstants.DEFAULT_TIMESTAMP_COLUMN
- + "') not specified in create table; list of columns is : "
- + properties.getProperty(serdeConstants.LIST_COLUMNS));
- }
- final List<PrimitiveTypeInfo>
- columnTypes =
- Utilities.getColumnTypes(properties)
- .stream()
- .map(TypeInfoFactory::getPrimitiveTypeInfo)
- .collect(Collectors.toList())
- .stream()
- .map(e -> e instanceof TimestampLocalTZTypeInfo ? tsTZTypeInfo : e)
- .collect(Collectors.toList());
- final List<ObjectInspector>
- inspectors =
- columnTypes.stream()
- .map(PrimitiveObjectInspectorFactory::getPrimitiveJavaObjectInspector)
- .collect(Collectors.toList());
- columns = columnNames.toArray(new String[0]);
- types = columnTypes.toArray(new PrimitiveTypeInfo[0]);
- inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
- }
-
- private void initFromDruidQueryPlan(Properties properties, String druidQuery) {
- Preconditions.checkNotNull(druidQuery, "Why Druid query is null");
- final List<ObjectInspector> inspectors = new ArrayList<>();
- final List<String> columnNames;
- final List<PrimitiveTypeInfo> columnTypes;
- final String
- fieldNamesProperty =
- Preconditions.checkNotNull(properties.getProperty(Constants.DRUID_QUERY_FIELD_NAMES, null));
- final String
- fieldTypesProperty =
- Preconditions.checkNotNull(properties.getProperty(Constants.DRUID_QUERY_FIELD_TYPES, null));
- if (fieldNamesProperty.isEmpty()) {
- // this might seem counter intuitive but some queries like query
- // SELECT YEAR(Calcs.date0) AS yr_date0_ok FROM druid_tableau.calcs Calcs WHERE (YEAR(Calcs.date0) IS NULL)
- // LIMIT 1
- // is planed in a way where we only push a filter down and keep the project of null as hive project. Thus empty
- // columns
- columnNames = Collections.emptyList();
- columnTypes = Collections.emptyList();
- } else {
- columnNames = Arrays.stream(fieldNamesProperty.trim().split(",")).collect(Collectors.toList());
- columnTypes =
- TypeInfoUtils.getTypeInfosFromTypeString(fieldTypesProperty)
- .stream()
- .map(e -> TypeInfoFactory.getPrimitiveTypeInfo(e.getTypeName()))
- .map(primitiveTypeInfo -> {
- if (primitiveTypeInfo instanceof TimestampLocalTZTypeInfo) {
- return tsTZTypeInfo;
- }
- return primitiveTypeInfo;
- })
- .collect(Collectors.toList());
- }
- columns = new String[columnNames.size()];
- types = new PrimitiveTypeInfo[columnNames.size()];
- for (int i = 0; i < columnTypes.size(); ++i) {
- columns[i] = columnNames.get(i);
- types[i] = columnTypes.get(i);
- inspectors.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(types[i]));
- }
- inspector = ObjectInspectorFactory.getStandardStructObjectInspector(columnNames, inspectors);
- }
-
- /* Submits the request and returns */
- protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query)
- throws SerDeException, IOException {
- InputStream response;
- try {
- response =
- DruidStorageHandlerUtils.submitRequest(DruidStorageHandler.getHttpClient(),
- DruidStorageHandlerUtils.createSmileRequest(address, query));
- } catch (Exception e) {
- throw new SerDeException(StringUtils.stringifyException(e));
- }
-
- // Retrieve results
- List<SegmentAnalysis> resultsList;
- try {
- // This will throw an exception in case of the response from druid is not an array
- // this case occurs if for instance druid query execution returns an exception instead of array of results.
- resultsList =
- DruidStorageHandlerUtils.SMILE_MAPPER.readValue(response, new TypeReference<List<SegmentAnalysis>>() {
- });
- } catch (Exception e) {
- response.close();
- throw new SerDeException(StringUtils.stringifyException(e));
- }
- if (resultsList == null || resultsList.isEmpty()) {
- throw new SerDeException("Connected to Druid but could not retrieve datasource information");
- }
- if (resultsList.size() != 1) {
- throw new SerDeException("Information about segments should have been merged");
- }
-
- return resultsList.get(0);
- }
-
- @Override public Class<? extends Writable> getSerializedClass() {
- return DruidWritable.class;
- }
-
- @Override public Writable serialize(Object o, ObjectInspector objectInspector) throws SerDeException {
- if (objectInspector.getCategory() != ObjectInspector.Category.STRUCT) {
- throw new SerDeException(getClass().toString()
- + " can only serialize struct types, but we got: "
- + objectInspector.getTypeName());
- }
-
- // Prepare the field ObjectInspectors
- StructObjectInspector soi = (StructObjectInspector) objectInspector;
- List<? extends StructField> fields = soi.getAllStructFieldRefs();
- List<Object> values = soi.getStructFieldsDataAsList(o);
- // We deserialize the result
- final Map<String, Object> value = new HashMap<>();
- for (int i = 0; i < columns.length; i++) {
- if (values.get(i) == null) {
- // null, we just add it
- value.put(columns[i], null);
- continue;
- }
- final Object res;
- switch (types[i].getPrimitiveCategory()) {
- case TIMESTAMP:
- res =
- ((TimestampObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i))
- .toEpochMilli();
- break;
- case TIMESTAMPLOCALTZ:
- res =
- ((TimestampLocalTZObjectInspector) fields.get(i)
- .getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i))
- .getZonedDateTime()
- .toInstant()
- .toEpochMilli();
- break;
- case BYTE:
- res = ((ByteObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case SHORT:
- res = ((ShortObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case INT:
- res = ((IntObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case LONG:
- res = ((LongObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case FLOAT:
- res = ((FloatObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case DOUBLE:
- res = ((DoubleObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i));
- break;
- case CHAR:
- res =
- ((HiveCharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i))
- .getValue();
- break;
- case VARCHAR:
- res =
- ((HiveVarcharObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i))
- .getValue();
- break;
- case STRING:
- res = ((StringObjectInspector) fields.get(i).getFieldObjectInspector()).getPrimitiveJavaObject(values.get(i));
- break;
- case BOOLEAN:
- res = ((BooleanObjectInspector) fields.get(i).getFieldObjectInspector()).get(values.get(i)) ? 1L : 0L;
- break;
- default:
- throw new SerDeException("Unsupported type: " + types[i].getPrimitiveCategory());
- }
- value.put(columns[i], res);
- }
- //Extract the partitions keys segments granularity and partition key if any
- // First Segment Granularity has to be here.
- final int granularityFieldIndex = columns.length;
- assert values.size() > granularityFieldIndex;
- Preconditions.checkArgument(fields.get(granularityFieldIndex)
- .getFieldName()
- .equals(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME));
-
- Timestamp
- timestamp =
- ((TimestampObjectInspector) fields.get(granularityFieldIndex).getFieldObjectInspector()).getPrimitiveJavaObject(
- values.get(granularityFieldIndex));
- Preconditions.checkNotNull(timestamp, "Timestamp column cannot have null value");
- value.put(Constants.DRUID_TIMESTAMP_GRANULARITY_COL_NAME, timestamp.toEpochMilli());
-
- if (values.size() == columns.length + 2) {
- // Then partition number if any.
- final int partitionNumPos = granularityFieldIndex + 1;
- Preconditions.checkArgument(fields.get(partitionNumPos).getFieldName().equals(Constants.DRUID_SHARD_KEY_COL_NAME),
- String.format("expecting to encounter %s but was %s",
- Constants.DRUID_SHARD_KEY_COL_NAME,
- fields.get(partitionNumPos).getFieldName()));
- value.put(Constants.DRUID_SHARD_KEY_COL_NAME,
- ((LongObjectInspector) fields.get(partitionNumPos)
- .getFieldObjectInspector()).get(values.get(partitionNumPos)));
- }
-
- return new DruidWritable(value);
- }
-
- /**
- * @param writable Druid Writable to be deserialized.
- * @return List of Hive Writables.
- * @throws SerDeException if there is Serde issues.
- */
- @Override public Object deserialize(Writable writable) throws SerDeException {
- final DruidWritable input = (DruidWritable) writable;
- final List<Object> output = Lists.newArrayListWithExpectedSize(columns.length);
- for (int i = 0; i < columns.length; i++) {
- final Object value = input.isCompacted() ? input.getCompactedValue().get(i) : input.getValue().get(columns[i]);
- if (value == null) {
- output.add(null);
- } else {
- output.add(convertAsPrimitive(value, types[i]));
- }
- }
- return output;
- }
-
- /**
- * Function to convert Druid Primitive values to Hive Primitives. Main usage of this is to pipe data to VectorRow.
- * This has the exact same logic as {@link DruidSerDe#deserialize(Writable)}, any modification here should be done
- * there as well.
- * Reason to have 2 function is that no vectorized path expects writables.
- *
- * @param writable Druid Writable.
- * @param rowBoat Rowboat used to carry columns values.
- * @throws SerDeException in case of deserialization errors.
- */
- public void deserializeAsPrimitive(Writable writable, final Object[] rowBoat) throws SerDeException {
- final DruidWritable input = (DruidWritable) writable;
- for (int i = 0; i < columns.length; i++) {
- final Object value = input.isCompacted() ? input.getCompactedValue().get(i) : input.getValue().get(columns[i]);
- rowBoat[i] = null;
- if (value != null) {
- rowBoat[i] = convertAsPrimitive(value, types[i]);
- }
- }
- }
-
- private static Object convertAsPrimitive(Object value, PrimitiveTypeInfo typeInfo) throws SerDeException {
- switch (typeInfo.getPrimitiveCategory()) {
- case TIMESTAMP:
- return Timestamp.ofEpochMilli(deserializeToMillis(value));
- case TIMESTAMPLOCALTZ:
- return new TimestampTZ(ZonedDateTime.ofInstant(Instant.ofEpochMilli(deserializeToMillis(value)),
- ((TimestampLocalTZTypeInfo) typeInfo).timeZone()));
- case DATE:
- return (Date.ofEpochMilli(deserializeToMillis(value)));
- case BYTE:
- return ((Number) value).byteValue();
- case SHORT:
- return ((Number) value).shortValue();
- case INT:
- if (value instanceof Number) {
- return ((Number) value).intValue();
- } else {
- // This is a corner case where we have an extract of time unit like day/month pushed as Extraction Fn
- //@TODO The best way to fix this is to add explicit output Druid types to Calcite Extraction Functions impls
- return (Integer.valueOf((String) value));
- }
- case LONG:
- return ((Number) value).longValue();
- case FLOAT:
- return ((Number) value).floatValue();
- case DOUBLE:
- return ((Number) value).doubleValue();
- case CHAR:
- return new HiveChar(value.toString(), ((CharTypeInfo) typeInfo).getLength());
- case VARCHAR:
- return new HiveVarchar(value.toString(), ((VarcharTypeInfo) typeInfo).getLength());
- case STRING:
- return value.toString();
- case BOOLEAN:
- if (value instanceof Number) {
- return (((Number) value).intValue() != 0);
- } else {
- return (Boolean.valueOf(value.toString()));
- }
- default:
- throw new SerDeException("Unknown type: " + typeInfo.getPrimitiveCategory());
- }
- }
-
- private static long deserializeToMillis(Object value) {
- long numberOfMillis;
- if (value instanceof Number) {
- numberOfMillis = ((Number) value).longValue();
- } else {
- // it is an extraction fn need to be parsed
- try {
- numberOfMillis = dateOptionalTimeParser().parseDateTime((String) value).getMillis();
- } catch (IllegalArgumentException e) {
- // we may not be able to parse the date if it already comes in Hive format,
- // we retry and otherwise fail
- numberOfMillis = Timestamp.valueOf((String) value).toEpochMilli();
- }
- }
- return numberOfMillis;
- }
-
- @Override public ObjectInspector getObjectInspector() {
- return inspector;
- }
-
- @Override public boolean shouldStoreFieldsInMetastore(Map<String, String> tableParams) {
- return true;
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
deleted file mode 100644
index 663d4c2484a..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidSerDeUtils.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Utils class for Druid SerDe.
- */
-final class DruidSerDeUtils {
- private DruidSerDeUtils() {
- }
-
- private static final Logger LOG = LoggerFactory.getLogger(DruidSerDeUtils.class);
-
- private static final String FLOAT_TYPE = "FLOAT";
- private static final String DOUBLE_TYPE = "DOUBLE";
- private static final String LONG_TYPE = "LONG";
- private static final String STRING_TYPE = "STRING";
-
- /* This method converts from the String representation of Druid type
- * to the corresponding Hive type */
- static PrimitiveTypeInfo convertDruidToHiveType(String typeName) {
- typeName = typeName.toUpperCase();
- switch (typeName) {
- case FLOAT_TYPE:
- return TypeInfoFactory.floatTypeInfo;
- case DOUBLE_TYPE:
- return TypeInfoFactory.doubleTypeInfo;
- case LONG_TYPE:
- return TypeInfoFactory.longTypeInfo;
- case STRING_TYPE:
- return TypeInfoFactory.stringTypeInfo;
- default:
- // This is a guard for special Druid types e.g. hyperUnique
- // (http://druid.io/docs/0.9.1.1/querying/aggregations.html#hyperunique-aggregator).
- // Currently, we do not support doing anything special with them in Hive.
- // However, those columns are there, and they can be actually read as normal
- // dimensions e.g. with a select query. Thus, we print the warning and just read them
- // as String.
- LOG.warn("Transformation to STRING for unknown type " + typeName);
- return TypeInfoFactory.stringTypeInfo;
- }
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java
deleted file mode 100644
index 95af15d2fa7..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTimeseriesQueryRecordReader.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JavaType;
-import org.apache.druid.query.Result;
-import org.apache.druid.query.timeseries.TimeseriesResultValue;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.conf.DruidConstants;
-import org.apache.hadoop.io.NullWritable;
-
-import java.io.IOException;
-
-/**
- * Record reader for results for Druid TimeseriesQuery.
- */
-public class DruidTimeseriesQueryRecordReader
- extends DruidQueryRecordReader<Result<TimeseriesResultValue>> {
-
- private static final TypeReference TYPE_REFERENCE = new TypeReference<Result<TimeseriesResultValue>>() {
- };
- private Result<TimeseriesResultValue> current;
-
- @Override
- protected JavaType getResultTypeDef() {
- return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE);
- }
-
- @Override
- public boolean nextKeyValue() {
- if (getQueryResultsIterator().hasNext()) {
- current = getQueryResultsIterator().next();
- return true;
- }
- return false;
- }
-
- @Override
- public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override
- public DruidWritable getCurrentValue() throws IOException, InterruptedException {
- // Create new value
- DruidWritable value = new DruidWritable(false);
- value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN,
- current.getTimestamp() == null ? null : current.getTimestamp().getMillis()
- );
- value.getValue().putAll(current.getValue().getBaseObject());
- return value;
- }
-
- @Override
- public boolean next(NullWritable key, DruidWritable value) {
- if (nextKeyValue()) {
- // Update value
- value.getValue().clear();
- value.getValue().put(DruidConstants.EVENT_TIMESTAMP_COLUMN,
- current.getTimestamp() == null ? null : current.getTimestamp().getMillis()
- );
- value.getValue().putAll(current.getValue().getBaseObject());
- return true;
- }
- return false;
- }
-
- @Override
- public float getProgress() throws IOException {
- return getQueryResultsIterator().hasNext() ? 0 : 1;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java
deleted file mode 100644
index 57c8b0ef146..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidTopNQueryRecordReader.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Iterator;
-
-import com.fasterxml.jackson.databind.JavaType;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.io.NullWritable;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-
-import org.apache.druid.query.Result;
-import org.apache.druid.query.topn.DimensionAndMetricValueExtractor;
-import org.apache.druid.query.topn.TopNResultValue;
-
-/**
- * Record reader for results for Druid TopNQuery.
- */
-public class DruidTopNQueryRecordReader extends DruidQueryRecordReader<Result<TopNResultValue>> {
-
- private static final TypeReference<Result<TopNResultValue>>
- TYPE_REFERENCE =
- new TypeReference<Result<TopNResultValue>>() {
- };
-
- private Result<TopNResultValue> current;
-
- private Iterator<DimensionAndMetricValueExtractor> values = Collections.emptyIterator();
-
- @Override protected JavaType getResultTypeDef() {
- return DruidStorageHandlerUtils.JSON_MAPPER.getTypeFactory().constructType(TYPE_REFERENCE);
- }
-
- @Override public boolean nextKeyValue() {
- if (values.hasNext()) {
- return true;
- }
- if (getQueryResultsIterator().hasNext()) {
- current = getQueryResultsIterator().next();
- values = current.getValue().getValue().iterator();
- return nextKeyValue();
- }
- return false;
- }
-
- @Override public NullWritable getCurrentKey() throws IOException, InterruptedException {
- return NullWritable.get();
- }
-
- @Override public DruidWritable getCurrentValue() throws IOException, InterruptedException {
- // Create new value
- DruidWritable value = new DruidWritable(false);
- value.getValue().put("timestamp", current.getTimestamp().getMillis());
- if (values.hasNext()) {
- value.getValue().putAll(values.next().getBaseObject());
- return value;
- }
- return value;
- }
-
- @Override public boolean next(NullWritable key, DruidWritable value) {
- if (nextKeyValue()) {
- // Update value
- value.getValue().clear();
- value.getValue().put("timestamp", current.getTimestamp().getMillis());
- if (values.hasNext()) {
- value.getValue().putAll(values.next().getBaseObject());
- }
- return true;
- }
- return false;
- }
-
- @Override public float getProgress() {
- return getQueryResultsIterator().hasNext() || values.hasNext() ? 0 : 1;
- }
-
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java
deleted file mode 100644
index b5850d46ac1..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/DruidWritable.java
+++ /dev/null
@@ -1,131 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.io.Writable;
-
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-
-/**
- * Writable for Druid results.
- */
-public class DruidWritable implements Writable {
-
- /**
- * value map stores column name to value mapping.
- * This is only used when the result is not compacted.
- */
- private final Map<String, Object> value;
-
- /**
- * list of values in a compacted form, Serializer/Deserializer needs to agree on the order of values.
- * This is only used when the result is compacted.
- */
- private transient List<Object> compactedValue;
-
- private final boolean compacted;
-
- public DruidWritable(boolean compacted) {
- this.compacted = compacted;
- if (compacted) {
- compactedValue = Lists.newArrayList();
- value = null;
- } else {
- value = new HashMap<>();
- compactedValue = null;
- }
- }
-
- public DruidWritable(Map<String, Object> value) {
- this.value = value;
- this.compactedValue = null;
- compacted = false;
- }
-
- public DruidWritable(List<Object> value) {
- this.compacted = true;
- this.compactedValue = value;
- this.value = null;
- }
-
- public Map<String, Object> getValue() {
- if (compacted) {
- throw new UnsupportedOperationException(
- "compacted DruidWritable does not support getValue(), use getCompactedValue()");
- }
- return value;
- }
-
- List<Object> getCompactedValue() {
- if (!compacted) {
- throw new UnsupportedOperationException(
- "non compacted DruidWritable does not support getCompactedValue(), use getValue()");
- }
- return compactedValue;
- }
-
- void setCompactedValue(List<Object> compactedValue) {
- this.compactedValue = compactedValue;
- }
-
- boolean isCompacted() {
- return compacted;
- }
-
- @Override public void write(DataOutput out) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override public void readFields(DataInput in) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override public int hashCode() {
- return Objects.hash(value, compactedValue, compacted);
- }
-
- @Override public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DruidWritable that = (DruidWritable) o;
- return compacted == that.compacted && Objects.equals(value, that.value) && Objects.equals(compactedValue,
- that.compactedValue);
- }
-
- @Override public String toString() {
- return "DruidWritable{"
- + "value="
- + value
- + ", compactedValue="
- + compactedValue
- + ", compacted="
- + compacted
- + '}';
- }
-}
diff --git a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java b/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java
deleted file mode 100644
index ba6a103af1f..00000000000
--- a/druid-handler/src/java/org/apache/hadoop/hive/druid/serde/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Package info file.
- */
-
-package org.apache.hadoop.hive.druid.serde;
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java
deleted file mode 100644
index 9119aa5e6b5..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/DerbyConnectorTestUtility.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid;
-
-import com.google.common.base.Suppliers;
-import org.apache.druid.metadata.MetadataStorageConnectorConfig;
-import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.metadata.storage.derby.DerbyConnector;
-import org.apache.druid.metadata.storage.derby.DerbyMetadataStorage;
-import org.junit.Assert;
-import org.junit.rules.ExternalResource;
-import org.skife.jdbi.v2.DBI;
-import org.skife.jdbi.v2.exceptions.UnableToObtainConnectionException;
-
-import java.sql.SQLException;
-import java.util.UUID;
-import java.util.function.Supplier;
-
-/**
- * Derby test class.
- */
-public class DerbyConnectorTestUtility extends DerbyConnector {
- private final String jdbcUri;
-
- public DerbyConnectorTestUtility(
- Supplier<MetadataStorageConnectorConfig> config,
- Supplier<MetadataStorageTablesConfig> dbTables
- ) {
- this(config, dbTables, "jdbc:derby:memory:druidTest" + dbSafeUUID());
- }
-
- protected DerbyConnectorTestUtility(
- Supplier<MetadataStorageConnectorConfig> config,
- Supplier<MetadataStorageTablesConfig> dbTables,
- String jdbcUri
- ) {
- super(new DerbyMetadataStorage(config.get()), config::get, dbTables::get, new DBI(jdbcUri + ";create=true"));
- this.jdbcUri = jdbcUri;
- }
-
- public void tearDown() {
- try {
- new DBI(jdbcUri + ";drop=true").open().close();
- } catch (UnableToObtainConnectionException e) {
- SQLException cause = (SQLException) e.getCause();
- // error code "08006" indicates proper shutdown
- Assert.assertEquals(String.format("Derby not shutdown: [%s]", cause.toString()), "08006",
- cause.getSQLState()
- );
- }
- }
-
- public static String dbSafeUUID() {
- return UUID.randomUUID().toString().replace("-", "");
- }
-
- public String getJdbcUri() {
- return jdbcUri;
- }
-
- /**
- * Derby connector rule.
- */
- public static class DerbyConnectorRule extends ExternalResource {
- private DerbyConnectorTestUtility connector;
-
- private final Supplier<MetadataStorageTablesConfig> dbTables;
-
- private final MetadataStorageConnectorConfig connectorConfig;
-
- public DerbyConnectorRule() {
- this("druidTest" + dbSafeUUID());
- }
-
- private DerbyConnectorRule(
- final String defaultBase
- ) {
- this(Suppliers.ofInstance(MetadataStorageTablesConfig.fromBase(defaultBase))::get);
- }
-
- public DerbyConnectorRule(
- Supplier<MetadataStorageTablesConfig> dbTables
- ) {
- this.dbTables = dbTables;
- this.connectorConfig = new MetadataStorageConnectorConfig() {
- @Override
- public String getConnectURI() {
- return connector.getJdbcUri();
- }
- };
- }
-
- @Override
- protected void before() throws Throwable {
- connector = new DerbyConnectorTestUtility(Suppliers.ofInstance(connectorConfig)::get, dbTables);
- connector.getDBI().open().close(); // create db
- }
-
- @Override
- protected void after() {
- connector.tearDown();
- }
-
- public DerbyConnectorTestUtility getConnector() {
- return connector;
- }
-
- public Supplier<MetadataStorageTablesConfig> metadataTablesConfigSupplier() {
- return dbTables;
- }
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java
deleted file mode 100644
index b47982f2a1f..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidQueryBasedInputFormatToAddFaultyHost.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import com.google.common.collect.Lists;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat;
-import org.apache.hadoop.hive.druid.io.HiveDruidSplit;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * This input format adds a faulty host as first input split location.
- * Tests should be able to query results successfully by trying next split locations.
- */
-public class QTestDruidQueryBasedInputFormatToAddFaultyHost extends DruidQueryBasedInputFormat {
-
- @Override
- protected HiveDruidSplit[] getInputSplits(Configuration conf) throws IOException {
- HiveDruidSplit[] inputSplits = super.getInputSplits(conf);
- List<HiveDruidSplit> list = new ArrayList<>();
- for(HiveDruidSplit split : inputSplits) {
- String[] locations = split.getLocations();
- List<String> locationsWithFaultyHost = Lists.newArrayListWithCapacity(locations.length + 1);
- // A non-queryable host location.
- locationsWithFaultyHost.add("localhost:8081");
- locationsWithFaultyHost.addAll(Arrays.asList(locations));
- HiveDruidSplit hiveDruidSplit = new HiveDruidSplit(split.getDruidQuery(), split.getPath(),
- locationsWithFaultyHost.toArray(new String[0])
- );
- list.add(hiveDruidSplit);
- }
- return list.toArray(new HiveDruidSplit[0]);
- }
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java
deleted file mode 100644
index 1764944ac04..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.druid.serde.DruidSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.util.StringUtils;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-
-import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
-import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
-
-/**
- * Druid SerDe to be used in tests.
- */
-public class QTestDruidSerDe extends DruidSerDe {
-
- // Request :
- // "{\"queryType\":\"segmentMetadata\",\"dataSource\":{\"type\":\"table\",\"name\":\"wikipedia\"},"
- // + "\"intervals\":{\"type\":\"intervals\","
- // + "\"intervals\":[\"-146136543-09-08T00:30:34.096-07:52:58/146140482-04-24T08:36:27.903-07:00\"]},"
- // + "\"toInclude\":{\"type\":\"all\"},\"merge\":true,\"context\":null,\"analysisTypes\":[],"
- // + "\"usingDefaultInterval\":true,\"lenientAggregatorMerge\":false,\"descending\":false}";
- private static final String
- RESPONSE =
- "[ {\r\n "
- + " \"id\" : \"merged\",\r\n "
- + " \"intervals\" : [ \"2010-01-01T00:00:00.000Z/2015-12-31T00:00:00.000Z\" ],\r\n "
- + " \"columns\" : {\r\n "
- + " \"__time\" : { \"type\" : \"LONG\", \"hasMultipleValues\" : false, \"size\" : 407240380, "
- + "\"cardinality\" : null, \"errorMessage\" : null },\r\n "
- + " \"robot\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\""
- + " : 1944, \"errorMessage\" : null },\r\n "
- + " \"namespace\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : true, \"size\" : 100000, "
- + "\"cardinality\" : 1504, \"errorMessage\" : null },\r\n "
- + " \"anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"unpatrolled\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"page\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": 1944, \"errorMessage\" : null },\r\n "
- + " \"language\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"newpage\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"user\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": 1944, \"errorMessage\" : null },\r\n "
- + " \"count\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"added\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"delta\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"variation\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : null, \"errorMessage\" : null },\r\n "
- + " \"deleted\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : null, \"errorMessage\" : null }\r\n "
- + " },\r\n "
- + " \"aggregators\" : {\r\n "
- + " \"count\" : { \"type\" : \"longSum\", \"name\" : \"count\", \"fieldName\" : \"count\" },\r\n "
- + " \"added\" : { \"type\" : \"doubleSum\", \"name\" : \"added\", \"fieldName\" : \"added\" },\r\n "
- + " \"delta\" : { \"type\" : \"doubleSum\", \"name\" : \"delta\", \"fieldName\" : \"delta\" },\r\n "
- + " \"variation\" : { \"type\" : \"doubleSum\", \"name\" : \"variation\", \"fieldName\" : \"variation\" },"
- + "\r\n "
- + " \"deleted\" : { \"type\" : \"doubleSum\", \"name\" : \"deleted\", \"fieldName\" : \"deleted\" }\r\n "
- + " },\r\n "
- + " \"queryGranularity\" : {\r\n \"type\": \"none\"\r\n },\r\n "
- + " \"size\" : 300000,\r\n "
- + " \"numRows\" : 5000000\r\n} ]";
-
- /* Submits the request and returns */
- @Override protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query)
- throws SerDeException {
- // Retrieve results
- List<SegmentAnalysis> resultsList;
- try {
- resultsList =
- DruidStorageHandlerUtils.JSON_MAPPER.readValue(RESPONSE, new TypeReference<List<SegmentAnalysis>>() {
- });
- } catch (Exception e) {
- throw new SerDeException(StringUtils.stringifyException(e));
- }
- return resultsList.get(0);
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java
deleted file mode 100644
index 55aa1c84b7a..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidSerDe2.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import java.util.List;
-
-import org.apache.hadoop.hive.druid.serde.DruidSerDe;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.util.StringUtils;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-
-import org.apache.druid.query.metadata.metadata.SegmentAnalysis;
-import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery;
-
-/**
- * Druid SerDe to be used in tests.
- */
-public class QTestDruidSerDe2 extends DruidSerDe {
-
- // Request :
- // "{\"queryType\":\"segmentMetadata\",\"dataSource\":{\"type\":\"table\",\"name\":\"wikipedia\"},"
- // + "\"intervals\":{\"type\":\"intervals\","
- // + "\"intervals\":[\"-146136543-09-08T00:30:34.096-07:52:58/146140482-04-24T08:36:27.903-07:00\"]},"
- // + "\"toInclude\":{\"type\":\"all\"},\"merge\":true,\"context\":null,\"analysisTypes\":[],"
- // + "\"usingDefaultInterval\":true,\"lenientAggregatorMerge\":false,\"descending\":false}";
- private static final String
- RESPONSE =
- "[ {\r\n "
- + " \"id\" : \"merged\",\r\n "
- + " \"intervals\" : [ \"2010-01-01T00:00:00.000Z/2015-12-31T00:00:00.000Z\" ],\r\n "
- + " \"columns\" : {\r\n "
- + " \"__time\" : { \"type\" : \"LONG\", \"hasMultipleValues\" : false, \"size\" : 407240380, "
- + "\"cardinality\" : null, \"errorMessage\" : null },\r\n "
- + " \"robot\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\""
- + " : 1944, \"errorMessage\" : null },\r\n "
- + " \"namespace\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : true, \"size\" : 100000, "
- + "\"cardinality\" : 1504, \"errorMessage\" : null },\r\n "
- + " \"anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- // Next column has a similar name as previous, but different casing.
- // This is allowed in Druid, but it should fail in Hive.
- + " \"Anonymous\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"page\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": 1944, \"errorMessage\" : null },\r\n "
- + " \"language\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"newpage\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : 1944, \"errorMessage\" : null },\r\n "
- + " \"user\" : { \"type\" : \"STRING\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": 1944, \"errorMessage\" : null },\r\n "
- + " \"count\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"added\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"delta\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, \"cardinality\" "
- + ": null, \"errorMessage\" : null },\r\n "
- + " \"variation\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : null, \"errorMessage\" : null },\r\n "
- + " \"deleted\" : { \"type\" : \"FLOAT\", \"hasMultipleValues\" : false, \"size\" : 100000, "
- + "\"cardinality\" : null, \"errorMessage\" : null }\r\n "
- + " },\r\n "
- + " \"aggregators\" : {\r\n "
- + " \"count\" : { \"type\" : \"longSum\", \"name\" : \"count\", \"fieldName\" : \"count\" },\r\n "
- + " \"added\" : { \"type\" : \"doubleSum\", \"name\" : \"added\", \"fieldName\" : \"added\" },\r\n "
- + " \"delta\" : { \"type\" : \"doubleSum\", \"name\" : \"delta\", \"fieldName\" : \"delta\" },\r\n "
- + " \"variation\" : { \"type\" : \"doubleSum\", \"name\" : \"variation\", \"fieldName\" : \"variation\" },"
- + "\r\n "
- + " \"deleted\" : { \"type\" : \"doubleSum\", \"name\" : \"deleted\", \"fieldName\" : \"deleted\" }\r\n "
- + " },\r\n "
- + " \"queryGranularity\" : {\r\n \"type\": \"none\"\r\n },\r\n "
- + " \"size\" : 300000,\r\n "
- + " \"numRows\" : 5000000\r\n} ]";
-
- /* Submits the request and returns */
- @Override protected SegmentAnalysis submitMetadataRequest(String address, SegmentMetadataQuery query)
- throws SerDeException {
- // Retrieve results
- List<SegmentAnalysis> resultsList;
- try {
- resultsList =
- DruidStorageHandlerUtils.JSON_MAPPER.readValue(RESPONSE, new TypeReference<List<SegmentAnalysis>>() {
- });
- } catch (Exception e) {
- throw new SerDeException(StringUtils.stringifyException(e));
- }
- return resultsList.get(0);
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java
deleted file mode 100644
index e17948c3fe6..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
-
-/**
- * Storage handler for Druid to be used in tests. It cannot connect to
- * Druid, and thus it cannot execute queries.
- */
-@SuppressWarnings("deprecation")
-public class QTestDruidStorageHandler extends DruidStorageHandler {
-
- @Override
- public Class<? extends AbstractSerDe> getSerDeClass() {
- return QTestDruidSerDe.class;
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java
deleted file mode 100644
index 9265a5d0a55..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandler2.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import org.apache.hadoop.hive.serde2.AbstractSerDe;
-
-/**
- * Storage handler for Druid to be used in tests. It cannot connect to
- * Druid, and thus it cannot execute queries.
- */
-@SuppressWarnings("deprecation")
-public class QTestDruidStorageHandler2 extends DruidStorageHandler {
-
- @Override
- public Class<? extends AbstractSerDe> getSerDeClass() {
- return QTestDruidSerDe2.class;
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java
deleted file mode 100644
index e0dec01e061..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/QTestDruidStorageHandlerToAddFaultyHost.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import org.apache.hadoop.mapred.InputFormat;
-
-/**
- * Storage handler for Druid to be used in tests.
- * It uses an input format that adds a faulty host as first input split location.
- * Tests should be able to query results successfully by trying next split locations.
- */
-@SuppressWarnings("deprecation")
-public class QTestDruidStorageHandlerToAddFaultyHost extends DruidStorageHandler {
-
- @Override
- public Class<? extends InputFormat> getInputFormatClass() {
- return QTestDruidQueryBasedInputFormatToAddFaultyHost.class;
- }
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
deleted file mode 100644
index 5040831b9a9..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestDruidStorageHandler.java
+++ /dev/null
@@ -1,774 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid;
-
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import org.apache.druid.indexer.JobHelper;
-import org.apache.druid.indexer.SQLMetadataStorageUpdaterJobHandler;
-import org.apache.druid.metadata.MetadataStorageTablesConfig;
-import org.apache.druid.segment.loading.DataSegmentPusher;
-import org.apache.druid.segment.loading.SegmentLoadingException;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusher;
-import org.apache.druid.storage.hdfs.HdfsDataSegmentPusherConfig;
-import org.apache.druid.timeline.DataSegment;
-import org.apache.druid.timeline.partition.LinearShardSpec;
-import org.apache.druid.timeline.partition.NoneShardSpec;
-import org.apache.druid.timeline.partition.ShardSpec;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
-import org.joda.time.DateTimeZone;
-import org.joda.time.Interval;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-import org.mockito.Mockito;
-import org.skife.jdbi.v2.Handle;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-/**
- * Test class for Druid Storage handler.
- */
-public class TestDruidStorageHandler {
-
- @Rule public final DerbyConnectorTestUtility.DerbyConnectorRule
- derbyConnectorRule =
- new DerbyConnectorTestUtility.DerbyConnectorRule();
-
- @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
-
- private static final String DB_NAME = "default";
- private static final String TABLE_NAME = "testName";
- private static final String DATA_SOURCE_NAME = "default.testName";
-
- private String segmentsTable;
-
- private String tableWorkingPath;
-
- private Configuration config;
-
- private DruidStorageHandler druidStorageHandler;
-
- private DataSegment createSegment(String location) throws IOException {
- return createSegment(location, new Interval(100, 170, DateTimeZone.UTC), "v1", new LinearShardSpec(0));
- }
-
- private DataSegment createSegment(String location, Interval interval, String version, ShardSpec shardSpec)
- throws IOException {
- FileUtils.writeStringToFile(new File(location), "dummySegmentData");
- return DataSegment.builder().dataSource(DATA_SOURCE_NAME).version(version).interval(interval).shardSpec(shardSpec)
- .loadSpec(ImmutableMap.of("path", location)).size(1000L)
- .build();
- }
-
- @Before public void before() throws Throwable {
- tableWorkingPath = temporaryFolder.newFolder().getAbsolutePath();
- segmentsTable = derbyConnectorRule.metadataTablesConfigSupplier().get().getSegmentsTable();
- Map<String, String> params = new HashMap<>();
- params.put("external.table.purge", "TRUE");
- Mockito.when(tableMock.getParameters()).thenReturn(params);
- Mockito.when(tableMock.getPartitionKeysSize()).thenReturn(0);
- StorageDescriptor storageDes = Mockito.mock(StorageDescriptor.class);
- Mockito.when(storageDes.getBucketColsSize()).thenReturn(0);
- Mockito.when(tableMock.getSd()).thenReturn(storageDes);
- Mockito.when(tableMock.getDbName()).thenReturn(DB_NAME);
- Mockito.when(tableMock.getTableName()).thenReturn(TABLE_NAME);
- config = new Configuration();
- config.set(String.valueOf(HiveConf.ConfVars.HIVE_QUERY_ID), "hive-" + UUID.randomUUID().toString());
- config.set(String.valueOf(HiveConf.ConfVars.DRUID_WORKING_DIR), tableWorkingPath);
- config.set(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY),
- new Path(tableWorkingPath, "finalSegmentDir").toString());
- config.set("hive.druid.maxTries", "0");
- druidStorageHandler =
- new DruidStorageHandler(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get());
- druidStorageHandler.setConf(config);
- }
-
- @After public void tearDown() {
- temporaryFolder.delete();
- }
-
- private final Table tableMock = Mockito.mock(Table.class);
-
- @Test public void testPreCreateTableWillCreateSegmentsTable() throws MetaException {
-
- try (Handle handle = derbyConnectorRule.getConnector().getDBI().open()) {
- Assert.assertFalse(derbyConnectorRule.getConnector().tableExists(handle, segmentsTable));
- druidStorageHandler.preCreateTable(tableMock);
- Assert.assertTrue(derbyConnectorRule.getConnector().tableExists(handle, segmentsTable));
- }
-
- }
-
- @Test(expected = MetaException.class) public void testPreCreateTableWhenDataSourceExists()
- throws MetaException, IOException {
- derbyConnectorRule.getConnector().createSegmentTable();
- SQLMetadataStorageUpdaterJobHandler
- sqlMetadataStorageUpdaterJobHandler =
- new SQLMetadataStorageUpdaterJobHandler(derbyConnectorRule.getConnector());
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- DataSegment dataSegment = createSegment(new Path(taskDirPath, "intermediatePath").toString());
-
- sqlMetadataStorageUpdaterJobHandler.publishSegments(segmentsTable,
- Collections.singletonList(dataSegment),
- DruidStorageHandlerUtils.JSON_MAPPER);
-
- druidStorageHandler.preCreateTable(tableMock);
- }
-
- @Test public void testCommitCreateTablePlusCommitDropTableWithoutPurge() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString());
-
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- druidStorageHandler.commitDropTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testCommitCreateTablePlusCommitDropTableWithPurge() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString());
-
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- druidStorageHandler.commitDropTable(tableMock, true);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testCommitCreateEmptyTablePlusCommitDropTableWithoutPurge() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- druidStorageHandler.commitDropTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testCommitCreateEmptyTablePlusCommitDropTableWithPurge() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- druidStorageHandler.commitDropTable(tableMock, true);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testCommitInsertTable() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString());
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testCommitEmptyInsertTable() throws MetaException, IOException {
- druidStorageHandler.preCreateTable(tableMock);
- druidStorageHandler.commitCreateTable(tableMock);
- Assert.assertArrayEquals(Lists.newArrayList().toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(derbyConnectorRule.getConnector(),
- derbyConnectorRule.metadataTablesConfigSupplier().get())).toArray());
- }
-
- @Test public void testDeleteSegment() throws IOException, SegmentLoadingException {
- String segmentRootPath = temporaryFolder.newFolder().getAbsolutePath();
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- DataSegment dataSegment = createSegment(new Path(taskDirPath, "index.zip").toString());
- HdfsDataSegmentPusherConfig hdfsDSPConfig = new HdfsDataSegmentPusherConfig();
- hdfsDSPConfig.setStorageDirectory(segmentRootPath);
- HdfsDataSegmentPusher
- hdfsDataSegmentPusher =
- new HdfsDataSegmentPusher(hdfsDSPConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
- Path
- segmentOutputPath =
- JobHelper.makeFileNamePath(new Path(segmentRootPath),
- localFileSystem,
- dataSegment,
- JobHelper.INDEX_ZIP,
- hdfsDataSegmentPusher);
- Path indexPath = new Path(segmentOutputPath, "index.zip");
- DataSegment
- dataSegmentWithLoadspect =
- DataSegment.builder(dataSegment).loadSpec(ImmutableMap.of("path", indexPath)).build();
- OutputStream outputStream = localFileSystem.create(indexPath, true);
- outputStream.close();
- Assert.assertTrue("index file is not created ??", localFileSystem.exists(indexPath));
- Assert.assertTrue(localFileSystem.exists(segmentOutputPath));
-
- druidStorageHandler.deleteSegment(dataSegmentWithLoadspect);
- // path format -- > .../dataSource/interval/version/partitionNum/xxx.zip
- Assert.assertFalse("Index file still there ??", localFileSystem.exists(indexPath));
- // path format of segmentOutputPath -- > .../dataSource/interval/version/partitionNum/
- Assert.assertFalse("PartitionNum directory still there ??", localFileSystem.exists(segmentOutputPath));
- Assert.assertFalse("Version directory still there ??", localFileSystem.exists(segmentOutputPath.getParent()));
- Assert.assertFalse("Interval directory still there ??",
- localFileSystem.exists(segmentOutputPath.getParent().getParent()));
- Assert.assertFalse("Data source directory still there ??",
- localFileSystem.exists(segmentOutputPath.getParent().getParent().getParent()));
- }
-
- @Test public void testCommitInsertOverwriteTable() throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY)));
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
-
- // This create and publish the segment to be overwritten
- List<DataSegment>
- existingSegments =
- Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)));
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
-
- // This creates and publish new segment
- DataSegment
- dataSegment =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(180, 250, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
-
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
-
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
-
- final List<DataSegment> dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig);
- Assert.assertEquals(1, dataSegmentList.size());
- DataSegment persistedSegment = Iterables.getOnlyElement(dataSegmentList);
- Assert.assertEquals(dataSegment, persistedSegment);
- Assert.assertEquals(dataSegment.getVersion(), persistedSegment.getVersion());
- Path
- expectedFinalHadoopPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP));
- Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()),
- persistedSegment.getLoadSpec());
- Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
- }
-
- @Test public void testCommitMultiInsertOverwriteTable() throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- druidStorageHandler.preCreateTable(tableMock);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY)));
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
-
- // This create and publish the segment to be overwritten
- List<DataSegment>
- existingSegments =
- Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)));
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
- // Check that there is one datasource with the published segment
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
-
- // Sequence is the following:
- // 1) INSERT with no segments -> Original segment still present in the datasource
- // 2) INSERT OVERWRITE with no segments -> Datasource is empty
- // 3) INSERT OVERWRITE with no segments -> Datasource is empty
- // 4) INSERT with no segments -> Datasource is empty
- // 5) INSERT with one segment -> Datasource has one segment
- // 6) INSERT OVERWRITE with one segment -> Datasource has one segment
- // 7) INSERT with one segment -> Datasource has two segments
- // 8) INSERT OVERWRITE with no segments -> Datasource is empty
-
- // We start:
- // #1
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
- Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #2
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #3
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #4
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #5
- DataSegment
- dataSegment1 =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(180, 250, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath1 =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment1,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment1, descriptorPath1);
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
- Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #6
- DataSegment
- dataSegment2 =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(200, 250, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath2 =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment2,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment2, descriptorPath2);
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
- Assert.assertEquals(1, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #7
- DataSegment
- dataSegment3 =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 200, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath3 =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment3,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment3, descriptorPath3);
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
- Assert.assertEquals(2, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
-
- // #8
- druidStorageHandler.commitInsertTable(tableMock, true);
- Assert.assertEquals(0, getUsedSegmentsList(connector, metadataStorageTablesConfig).size());
- }
-
- private List<DataSegment> getUsedSegmentsList(DerbyConnectorTestUtility connector,
- final MetadataStorageTablesConfig metadataStorageTablesConfig) {
- return connector.getDBI()
- .withHandle(handle -> handle.createQuery(String.format(
- "SELECT payload FROM %s WHERE used=true ORDER BY created_date ASC",
- metadataStorageTablesConfig.getSegmentsTable()))
- .map((i, resultSet, statementContext) -> {
- try {
- return DruidStorageHandlerUtils.JSON_MAPPER.readValue(resultSet.getBytes("payload"), DataSegment.class);
- } catch (IOException e) {
- throw Throwables.propagate(e);
- }
- }).list());
- }
-
- @Test public void testCommitInsertIntoTable() throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- List<DataSegment>
- existingSegments =
- Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(1)));
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY)));
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
-
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
- DataSegment
- dataSegment =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
-
- final List<DataSegment> dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig);
- Assert.assertEquals(2, dataSegmentList.size());
-
- DataSegment persistedSegment = dataSegmentList.get(1);
- // Insert into appends to old version
- Assert.assertEquals("v0", persistedSegment.getVersion());
- Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec);
- Assert.assertEquals(2, persistedSegment.getShardSpec().getPartitionNum());
-
- Path
- expectedFinalHadoopPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP));
-
- Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()),
- persistedSegment.getLoadSpec());
- Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
- }
-
- @Test public void testInsertIntoAppendOneMorePartition() throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY)));
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
-
- List<DataSegment>
- existingSegments =
- Collections.singletonList(createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)));
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
-
- DataSegment
- dataSegment =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0));
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(), Lists.newArrayList(
- DruidStorageHandlerUtils.getAllDataSourceNames(connector,
- metadataStorageTablesConfig
- )).toArray());
-
- final List<DataSegment> dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig);
- Assert.assertEquals(2, dataSegmentList.size());
-
- DataSegment persistedSegment = dataSegmentList.get(1);
- Assert.assertEquals("v0", persistedSegment.getVersion());
- Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec);
- Assert.assertEquals(1, persistedSegment.getShardSpec().getPartitionNum());
-
- Path
- expectedFinalHadoopPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP));
-
- Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()),
- persistedSegment.getLoadSpec());
- Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
- }
-
- @Test public void testCommitInsertIntoWhenDestinationSegmentFileExist() throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- List<DataSegment>
- existingSegments =
- Collections.singletonList(createSegment(new Path(taskDirPath, "index_old.zip").toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(1)));
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(config.get(String.valueOf(HiveConf.ConfVars.DRUID_SEGMENT_DIRECTORY)));
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
- DataSegment
- dataSegment =
- createSegment(new Path(taskDirPath, "index.zip").toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(dataSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, dataSegment, descriptorPath);
-
- // Create segment file at the destination location with LinearShardSpec(2)
- DataSegment
- segment =
- createSegment(new Path(taskDirPath, "index_conflict.zip").toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(1));
- Path
- segmentPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(segment, DruidStorageHandlerUtils.INDEX_ZIP));
- FileUtils.writeStringToFile(new File(segmentPath.toUri()), "dummy");
-
- druidStorageHandler.commitInsertTable(tableMock, false);
- Assert.assertArrayEquals(Lists.newArrayList(DATA_SOURCE_NAME).toArray(),
- Lists.newArrayList(DruidStorageHandlerUtils.getAllDataSourceNames(connector, metadataStorageTablesConfig))
- .toArray());
-
- final List<DataSegment> dataSegmentList = getUsedSegmentsList(connector, metadataStorageTablesConfig);
- Assert.assertEquals(2, dataSegmentList.size());
-
- DataSegment persistedSegment = dataSegmentList.get(1);
- // Insert into appends to old version
- Assert.assertEquals("v0", persistedSegment.getVersion());
- Assert.assertTrue(persistedSegment.getShardSpec() instanceof LinearShardSpec);
- // insert into should skip and increment partition number to 3
- Assert.assertEquals(2, persistedSegment.getShardSpec().getPartitionNum());
- Path
- expectedFinalHadoopPath =
- new Path(dataSegmentPusher.getPathForHadoop(),
- dataSegmentPusher.makeIndexPathName(persistedSegment, DruidStorageHandlerUtils.INDEX_ZIP));
-
- Assert.assertEquals(ImmutableMap.of("type", "hdfs", "path", expectedFinalHadoopPath.toString()),
- persistedSegment.getLoadSpec());
- Assert.assertEquals("dummySegmentData", FileUtils.readFileToString(new File(expectedFinalHadoopPath.toUri())));
- }
-
- @Test(expected = MetaException.class) public void testCommitInsertIntoWithConflictingIntervalSegment()
- throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- List<DataSegment>
- existingSegments =
- Arrays.asList(createSegment(new Path(taskDirPath, "index_old_1.zip").toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)),
- createSegment(new Path(taskDirPath, "index_old_2.zip").toString(),
- new Interval(150, 200, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)),
- createSegment(new Path(taskDirPath, "index_old_3.zip").toString(),
- new Interval(200, 300, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)));
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(taskDirPath.toString());
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
-
- // Try appending segment with conflicting interval
- DataSegment
- conflictingSegment =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 300, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(conflictingSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, conflictingSegment, descriptorPath);
- druidStorageHandler.commitInsertTable(tableMock, false);
- }
-
- @Test(expected = MetaException.class) public void testCommitInsertIntoWithNonExtendableSegment()
- throws MetaException, IOException {
- DerbyConnectorTestUtility connector = derbyConnectorRule.getConnector();
- MetadataStorageTablesConfig metadataStorageTablesConfig = derbyConnectorRule.metadataTablesConfigSupplier().get();
- druidStorageHandler.preCreateTable(tableMock);
- LocalFileSystem localFileSystem = FileSystem.getLocal(config);
- Path taskDirPath = new Path(tableWorkingPath, druidStorageHandler.makeStagingName());
- List<DataSegment>
- existingSegments =
- Arrays.asList(createSegment(new Path(taskDirPath, "index_old_1.zip").toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v0",
- NoneShardSpec.instance()),
- createSegment(new Path(taskDirPath, "index_old_2.zip").toString(),
- new Interval(200, 250, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)),
- createSegment(new Path(taskDirPath, "index_old_3.zip").toString(),
- new Interval(250, 300, DateTimeZone.UTC),
- "v0",
- new LinearShardSpec(0)));
- HdfsDataSegmentPusherConfig pusherConfig = new HdfsDataSegmentPusherConfig();
- pusherConfig.setStorageDirectory(taskDirPath.toString());
- DataSegmentPusher
- dataSegmentPusher =
- new HdfsDataSegmentPusher(pusherConfig, config, DruidStorageHandlerUtils.JSON_MAPPER);
- DruidStorageHandlerUtils.publishSegmentsAndCommit(connector,
- metadataStorageTablesConfig,
- DATA_SOURCE_NAME,
- existingSegments,
- true,
- config,
- dataSegmentPusher);
-
- // Try appending to non extendable shard spec
- DataSegment
- conflictingSegment =
- createSegment(new Path(taskDirPath, DruidStorageHandlerUtils.INDEX_ZIP).toString(),
- new Interval(100, 150, DateTimeZone.UTC),
- "v1",
- new LinearShardSpec(0));
- Path
- descriptorPath =
- DruidStorageHandlerUtils.makeSegmentDescriptorOutputPath(conflictingSegment,
- new Path(taskDirPath, DruidStorageHandler.SEGMENTS_DESCRIPTOR_DIR_NAME));
- DruidStorageHandlerUtils.writeSegmentDescriptor(localFileSystem, conflictingSegment, descriptorPath);
-
- druidStorageHandler.commitInsertTable(tableMock, false);
-
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
deleted file mode 100644
index 2bcbb14f9fb..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/TestHiveDruidQueryBasedInputFormat.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid;
-
-import java.lang.reflect.Method;
-import java.util.Arrays;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.conf.HiveConf;
-import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat;
-import org.apache.hadoop.hive.druid.io.HiveDruidSplit;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-
-import org.apache.druid.query.Query;
-import static org.junit.Assert.assertEquals;
-import org.junit.Test;
-
-/**
- * Test Class.
- */
-@SuppressWarnings("SameParameterValue") public class TestHiveDruidQueryBasedInputFormat {
-
- private static final String
- TIMESERIES_QUERY =
- "{ \"queryType\": \"timeseries\", "
- + " \"dataSource\": \"sample_datasource\", "
- + " \"granularity\": \"DAY\", "
- + " \"descending\": \"true\", "
- + " \"intervals\": [ \"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\" ]}";
- private static final String
- TIMESERIES_QUERY_SPLIT =
- "[HiveDruidSplit{{\"queryType\":\"timeseries\","
- + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"},"
- + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":"
- + "[\"2012-01-01T08:00:00.000Z/2012-01-03T08:00:00.000Z\"]},"
- + "\"descending\":true,"
- + "\"virtualColumns\":[],"
- + "\"filter\":null,"
- + "\"granularity\":\"DAY\","
- + "\"aggregations\":[],"
- + "\"postAggregations\":[],"
- + "\"limit\":2147483647,"
- + "\"context\":{\"queryId\":\"\"}}, [localhost:8082]}]";
-
- private static final String
- TOPN_QUERY =
- "{ \"queryType\": \"topN\", "
- + " \"dataSource\": \"sample_data\", "
- + " \"dimension\": \"sample_dim\", "
- + " \"threshold\": 5, "
- + " \"metric\": \"count\", "
- + " \"aggregations\": [ "
- + " { "
- + " \"type\": \"longSum\", "
- + " \"name\": \"count\", "
- + " \"fieldName\": \"count\" "
- + " }, "
- + " { "
- + " \"type\": \"doubleSum\", "
- + " \"name\": \"some_metric\", "
- + " \"fieldName\": \"some_metric\" "
- + " } "
- + " ], "
- + " \"granularity\": \"all\", "
- + " \"intervals\": [ "
- + " \"2013-08-31T00:00:00.000-07:00/2013-09-03T00:00:00.000-07:00\" "
- + " ]}";
- private static final String
- TOPN_QUERY_SPLIT =
- "[HiveDruidSplit{{\"queryType\":\"topN\","
- + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_data\"},"
- + "\"virtualColumns\":[],"
- + "\"dimension\":{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"sample_dim\","
- + "\"outputName\":\"sample_dim\",\"outputType\":\"STRING\"},"
- + "\"metric\":{\"type\":\"LegacyTopNMetricSpec\",\"metric\":\"count\"},"
- + "\"threshold\":5,"
- + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2013-08-31T07:00:00"
- + ".000Z/2013-09-03T07:00:00.000Z\"]},"
- + "\"filter\":null,"
- + "\"granularity\":{\"type\":\"all\"},"
- + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"count\",\"fieldName\":\"count\",\"expression\":null},"
- + "{\"type\":\"doubleSum\",\"name\":\"some_metric\",\"fieldName\":\"some_metric\",\"expression\":null}],"
- + "\"postAggregations\":[],"
- + "\"context\":{\"queryId\":\"\"},"
- + "\"descending\":false}, [localhost:8082]}]";
-
- private static final String
- GROUP_BY_QUERY =
- "{ \"queryType\": \"groupBy\", "
- + " \"dataSource\": \"sample_datasource\", "
- + " \"granularity\": \"day\", "
- + " \"dimensions\": [\"country\", \"device\"], "
- + " \"limitSpec\": {"
- + " \"type\": \"default\","
- + " \"limit\": 5000,"
- + " \"columns\": [\"country\", \"data_transfer\"] }, "
- + " \"aggregations\": [ "
- + " { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" }, "
- + " { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" } "
- + " ], "
- + " \"intervals\": [ \"2012-01-01T00:00:00.000-08:00/2012-01-03T00:00:00.000-08:00\" ]"
- + " }";
- private static final String
- GROUP_BY_QUERY_SPLIT =
- "[HiveDruidSplit{{\"queryType\":\"groupBy\","
- + "\"dataSource\":{\"type\":\"table\",\"name\":\"sample_datasource\"},"
- + "\"intervals\":{\"type\":\"LegacySegmentSpec\",\"intervals\":[\"2012-01-01T08:00:00"
- + ".000Z/2012-01-03T08:00:00.000Z\"]},"
- + "\"virtualColumns\":[],"
- + "\"filter\":null,"
- + "\"granularity\":\"DAY\","
- + "\"dimensions\":[{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"country\",\"outputName\":\"country\","
- + "\"outputType\":\"STRING\"},"
- + "{\"type\":\"LegacyDimensionSpec\",\"dimension\":\"device\",\"outputName\":\"device\","
- + "\"outputType\":\"STRING\"}],"
- + "\"aggregations\":[{\"type\":\"longSum\",\"name\":\"total_usage\",\"fieldName\":\"user_count\","
- + "\"expression\":null},"
- + "{\"type\":\"doubleSum\",\"name\":\"data_transfer\",\"fieldName\":\"data_transfer\",\"expression\":null}],"
- + "\"postAggregations\":[],"
- + "\"having\":null,"
- + "\"limitSpec\":{\"type\":\"default\",\"columns\":[{\"dimension\":\"country\",\"direction\":\"ascending\","
- + "\"dimensionOrder\":{\"type\":\"lexicographic\"}},"
- + "{\"dimension\":\"data_transfer\",\"direction\":\"ascending\","
- + "\"dimensionOrder\":{\"type\":\"lexicographic\"}}],\"limit\":5000},"
- + "\"context\":{\"queryId\":\"\"},"
- + "\"descending\":false}, [localhost:8082]}]";
-
- @Test
- public void testTimeZone() throws Exception {
- DruidQueryBasedInputFormat input = new DruidQueryBasedInputFormat();
-
- Method method1 = DruidQueryBasedInputFormat.class.getDeclaredMethod("getInputSplits", Configuration.class);
- method1.setAccessible(true);
-
- // Create, initialize, and test
- Configuration conf = createPropertiesQuery("sample_datasource", Query.TIMESERIES, TIMESERIES_QUERY);
- HiveDruidSplit[] resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
- assertEquals(TIMESERIES_QUERY_SPLIT, Arrays.toString(resultSplits));
-
- conf = createPropertiesQuery("sample_datasource", Query.TOPN, TOPN_QUERY);
- resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
- assertEquals(TOPN_QUERY_SPLIT, Arrays.toString(resultSplits));
-
- conf = createPropertiesQuery("sample_datasource", Query.GROUP_BY, GROUP_BY_QUERY);
- resultSplits = (HiveDruidSplit[]) method1.invoke(input, conf);
- assertEquals(GROUP_BY_QUERY_SPLIT, Arrays.toString(resultSplits));
-
- }
-
- private static Configuration createPropertiesQuery(String dataSource, String queryType, String jsonQuery) {
- Configuration conf = new Configuration();
- // Set the configuration parameters
- conf.set(FileInputFormat.INPUT_DIR, "/my/dir");
- conf.set(HiveConf.ConfVars.HIVE_DRUID_BROKER_DEFAULT_ADDRESS.varname, "localhost:8082");
- conf.set(Constants.DRUID_DATA_SOURCE, dataSource);
- conf.set(Constants.DRUID_QUERY_JSON, jsonQuery);
- conf.set(Constants.DRUID_QUERY_TYPE, queryType);
- return conf;
- }
-
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java
deleted file mode 100644
index be3ff6fee11..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/io/TestHiveDruidSplit.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.druid.io;
-
-import org.apache.hadoop.fs.Path;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-/**
- * Test Class for input splits.
- */
-public class TestHiveDruidSplit {
- @Test public void testSerDeser() throws IOException {
- HiveDruidSplit
- hiveDruidSplit =
- new HiveDruidSplit("query string", new Path("test-path"), new String[] {"host:8080", "host2:8090"});
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- DataOutput dataOutput = new DataOutputStream(byteArrayOutputStream);
- hiveDruidSplit.write(dataOutput);
- ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
- HiveDruidSplit actualHiveDruidSplit = new HiveDruidSplit();
- actualHiveDruidSplit.readFields(new DataInputStream(byteArrayInputStream));
- Assert.assertEquals(actualHiveDruidSplit.getDruidQuery(), "query string");
- Assert.assertArrayEquals(actualHiveDruidSplit.getLocations(), new String[] {"host:8080", "host2:8090"});
- }
-}
diff --git a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java b/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
deleted file mode 100644
index 5157e4656c4..00000000000
--- a/druid-handler/src/test/org/apache/hadoop/hive/druid/serde/TestDruidSerDe.java
+++ /dev/null
@@ -1,884 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hive.druid.serde;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.lang.reflect.InvocationTargetException;
-import java.time.Instant;
-import java.time.ZoneOffset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Properties;
-import java.util.stream.Collectors;
-
-import org.apache.druid.java.util.http.client.HttpClient;
-import org.apache.druid.java.util.http.client.response.HttpResponseHandler;
-import org.apache.druid.query.scan.ScanResultValue;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.common.type.Timestamp;
-import org.apache.hadoop.hive.common.type.TimestampTZ;
-import org.apache.hadoop.hive.conf.Constants;
-import org.apache.hadoop.hive.druid.DruidStorageHandlerUtils;
-import org.apache.hadoop.hive.druid.QTestDruidSerDe;
-import org.apache.hadoop.hive.druid.io.DruidQueryBasedInputFormat;
-import org.apache.hadoop.hive.druid.io.HiveDruidSplit;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.io.ByteWritable;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
-import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
-import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Text;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.SettableFuture;
-
-import org.apache.druid.data.input.Row;
-import org.apache.druid.query.Query;
-import org.apache.druid.query.Result;
-import org.apache.druid.query.timeseries.TimeseriesResultValue;
-import org.apache.druid.query.topn.TopNResultValue;
-import org.junit.rules.ExpectedException;
-
-/**
- * Basic tests for Druid SerDe. The examples are taken from Druid 0.9.1.1
- * documentation.
- */
-@SuppressWarnings({"SameParameterValue", "SpellCheckingInspection"})
-public class TestDruidSerDe {
- // Timeseries query
- private static final String
- TIMESERIES_QUERY =
- "{ \"queryType\": \"timeseries\", "
- + " \"dataSource\": \"sample_datasource\", "
- + " \"granularity\": \"day\", "
- + " \"descending\": \"true\", "
- + " \"filter\": { "
- + " \"type\": \"and\", "
- + " \"fields\": [ "
- + " { \"type\": \"selector\", \"dimension\": \"sample_dimension1\", \"value\": \"sample_value1\" }, "
- + " { \"type\": \"or\", "
- + " \"fields\": [ "
- + " { \"type\": \"selector\", \"dimension\": \"sample_dimension2\", \"value\": \"sample_value2\" }, "
- + " { \"type\": \"selector\", \"dimension\": \"sample_dimension3\", \"value\": \"sample_value3\" } "
- + " ] "
- + " } "
- + " ] "
- + " }, "
- + " \"aggregations\": [ "
- + " { \"type\": \"longSum\", \"name\": \"sample_name1\", \"fieldName\": \"sample_fieldName1\" }, "
- + " { \"type\": \"doubleSum\", \"name\": \"sample_name2\", \"fieldName\": \"sample_fieldName2\" } "
- + " ], "
- + " \"postAggregations\": [ "
- + " { \"type\": \"arithmetic\", "
- + " \"name\": \"sample_divide\", "
- + " \"fn\": \"/\", "
- + " \"fields\": [ "
- + " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name1\", \"fieldName\": \"sample_name1\" "
- + "}, "
- + " { \"type\": \"fieldAccess\", \"name\": \"postAgg__sample_name2\", \"fieldName\": \"sample_name2\" } "
- + " ] "
- + " } "
- + " ], "
- + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ]}";
-
- // Timeseries query results
- private static final String
- TIMESERIES_QUERY_RESULTS =
- "[ "
- + "{ "
- + " \"timestamp\": \"2012-01-01T00:00:00.000Z\", "
- + " \"result\": { \"sample_name1\": 0, \"sample_name2\": 1.0, \"sample_divide\": 2.2222 } "
- + "}, "
- + "{ "
- + " \"timestamp\": \"2012-01-02T00:00:00.000Z\", "
- + " \"result\": { \"sample_name1\": 2, \"sample_name2\": 3.32, \"sample_divide\": 4 } "
- + "}]";
-
- private byte[] tsQueryResults;
- private byte[] topNQueryResults;
- private byte[] groupByQueryResults;
- private byte[] groupByTimeExtractQueryResults;
- private byte[] selectQueryResults;
- private byte[] groupByMonthExtractQueryResults;
- private byte[] scanQueryResults;
-
- // Timeseries query results as records
- private static final Object[][] TIMESERIES_QUERY_RESULTS_RECORDS = new Object[][]{
- new Object[]{
- new TimestampTZ(Instant.ofEpochMilli(1325376000000L).atZone(ZoneOffset.UTC)),
- 0L,
- 1.0F,
- 2.2222F},
- new Object[]{
- new TimestampTZ(Instant.ofEpochMilli(1325462400000L).atZone(ZoneOffset.UTC)),
- 2L,
- 3.32F,
- 4F}};
-
- // Timeseries query results as records (types defined by metastore)
- private static final String TIMESERIES_COLUMN_NAMES = "timestamp,sample_name1,sample_name2,sample_divide";
- private static final String TIMESERIES_COLUMN_TYPES = "timestamp with local time zone,bigint,float,float";
- // TopN query
- private static final String
- TOPN_QUERY =
- "{ \"queryType\": \"topN\", "
- + " \"dataSource\": \"sample_data\", "
- + " \"dimension\": \"sample_dim\", "
- + " \"threshold\": 5, "
- + " \"metric\": \"count\", "
- + " \"granularity\": \"all\", "
- + " \"filter\": { "
- + " \"type\": \"and\", "
- + " \"fields\": [ "
- + " { "
- + " \"type\": \"selector\", "
- + " \"dimension\": \"dim1\", "
- + " \"value\": \"some_value\" "
- + " }, "
- + " { "
- + " \"type\": \"selector\", "
- + " \"dimension\": \"dim2\", "
- + " \"value\": \"some_other_val\" "
- + " } "
- + " ] "
- + " }, "
- + " \"aggregations\": [ "
- + " { "
- + " \"type\": \"longSum\", "
- + " \"name\": \"count\", "
- + " \"fieldName\": \"count\" "
- + " }, "
- + " { "
- + " \"type\": \"doubleSum\", "
- + " \"name\": \"some_metric\", "
- + " \"fieldName\": \"some_metric\" "
- + " } "
- + " ], "
- + " \"postAggregations\": [ "
- + " { "
- + " \"type\": \"arithmetic\", "
- + " \"name\": \"sample_divide\", "
- + " \"fn\": \"/\", "
- + " \"fields\": [ "
- + " { "
- + " \"type\": \"fieldAccess\", "
- + " \"name\": \"some_metric\", "
- + " \"fieldName\": \"some_metric\" "
- + " }, "
- + " { "
- + " \"type\": \"fieldAccess\", "
- + " \"name\": \"count\", "
- + " \"fieldName\": \"count\" "
- + " } "
- + " ] "
- + " } "
- + " ], "
- + " \"intervals\": [ "
- + " \"2013-08-31T00:00:00.000/2013-09-03T00:00:00.000\" "
- + " ]}";
-
- // TopN query results
- private static final String
- TOPN_QUERY_RESULTS =
- "[ "
- + " { "
- + " \"timestamp\": \"2013-08-31T00:00:00.000Z\", "
- + " \"result\": [ "
- + " { "
- + " \"sample_dim\": \"dim1_val\", "
- + " \"count\": 111, "
- + " \"some_metric\": 10669, "
- + " \"sample_divide\": 96.11711711711712 "
- + " }, "
- + " { "
- + " \"sample_dim\": \"another_dim1_val\", "
- + " \"count\": 88, "
- + " \"some_metric\": 28344, "
- + " \"sample_divide\": 322.09090909090907 "
- + " }, "
- + " { "
- + " \"sample_dim\": \"dim1_val3\", "
- + " \"count\": 70, "
- + " \"some_metric\": 871, "
- + " \"sample_divide\": 12.442857142857143 "
- + " }, "
- + " { "
- + " \"sample_dim\": \"dim1_val4\", "
- + " \"count\": 62, "
- + " \"some_metric\": 815, "
- + " \"sample_divide\": 13.14516129032258 "
- + " }, "
- + " { "
- + " \"sample_dim\": \"dim1_val5\", "
- + " \"count\": 60, "
- + " \"some_metric\": 2787, "
- + " \"sample_divide\": 46.45 "
- + " } "
- + " ] "
- + " }]";
-
- // TopN query results as records
- private static final Object[][] TOPN_QUERY_RESULTS_RECORDS = new Object[][]{
- new Object[]{
- new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC)),
- "dim1_val", 111L, 10669F, 96.11711711711712F},
- new Object[]{
- (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
- "another_dim1_val", 88L, 28344F, 322.09090909090907F},
- new Object[]{
- (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
- "dim1_val3", 70L, 871F, 12.442857142857143F},
- new Object[]{
- new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC)),
- "dim1_val4", 62L, 815F, 13.14516129032258F},
- new Object[]{
- (new TimestampTZ(Instant.ofEpochMilli(1377907200000L).atZone(ZoneOffset.UTC))),
- "dim1_val5", 60L, 2787F, 46.45F}};
-
- // TopN query results as records (types defined by metastore)
- private static final String TOPN_COLUMN_NAMES = "timestamp,sample_dim,count,some_metric,sample_divide";
- private static final String TOPN_COLUMN_TYPES = "timestamp with local time zone,string,bigint,float,float";
-
- // GroupBy query
- private static final String
- GROUP_BY_QUERY =
- "{ "
- + " \"queryType\": \"groupBy\", "
- + " \"dataSource\": \"sample_datasource\", "
- + " \"granularity\": \"day\", "
- + " \"dimensions\": [\"country\", \"device\"], "
- + " \"limitSpec\": {"
- + " \"type\": \"default\","
- + " \"limit\": 5000,"
- + " \"columns\": [\"country\", \"data_transfer\"] }, "
- + " \"filter\": { "
- + " \"type\": \"and\", "
- + " \"fields\": [ "
- + " { \"type\": \"selector\", \"dimension\": \"carrier\", \"value\": \"AT&T\" }, "
- + " { \"type\": \"or\", "
- + " \"fields\": [ "
- + " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Apple\" }, "
- + " { \"type\": \"selector\", \"dimension\": \"make\", \"value\": \"Samsung\" } "
- + " ] "
- + " } "
- + " ] "
- + " }, "
- + " \"aggregations\": [ "
- + " { \"type\": \"longSum\", \"name\": \"total_usage\", \"fieldName\": \"user_count\" }, "
- + " { \"type\": \"doubleSum\", \"name\": \"data_transfer\", \"fieldName\": \"data_transfer\" } "
- + " ], "
- + " \"postAggregations\": [ "
- + " { \"type\": \"arithmetic\", "
- + " \"name\": \"avg_usage\", "
- + " \"fn\": \"/\", "
- + " \"fields\": [ "
- + " { \"type\": \"fieldAccess\", \"fieldName\": \"data_transfer\" }, "
- + " { \"type\": \"fieldAccess\", \"fieldName\": \"total_usage\" } "
- + " ] "
- + " } "
- + " ], "
- + " \"intervals\": [ \"2012-01-01T00:00:00.000/2012-01-03T00:00:00.000\" ], "
- + " \"having\": { "
- + " \"type\": \"greaterThan\", "
- + " \"aggregation\": \"total_usage\", "
- + " \"value\": 100 "
- + " }}";
-
- // GroupBy query results
- private static final String
- GROUP_BY_QUERY_RESULTS =
- "[ "
- + " { "
- + " \"version\" : \"v1\", "
- + " \"timestamp\" : \"2012-01-01T00:00:00.000Z\", "
- + " \"event\" : { "
- + " \"country\" : \"India\", "
- + " \"device\" : \"phone\", "
- + " \"total_usage\" : 88, "
- + " \"data_transfer\" : 29.91233453, "
- + " \"avg_usage\" : 60.32 "
- + " } "
- + " }, "
- + " { "
- + " \"version\" : \"v1\", "
- + " \"timestamp\" : \"2012-01-01T00:00:12.000Z\", "
- + " \"event\" : { "
- + " \"country\" : \"Spain\", "
- + " \"device\" : \"pc\", "
... 21787 lines suppressed ...