You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/10/18 16:43:55 UTC
[32/32] incubator-impala git commit: Remove Llama dependency
Remove Llama dependency
This change prevents us from depending on LLAMA to build.
Note that the LLAMA MiniKDC is left in - it is a test
utility that does not depend on LLAMA itself.
IMPALA-4292 tracks cleaning this up.
Testing:
Ran a private build to verify that all tests pass.
Change-Id: If2e5e21d8047097d56062ded11b0832a1d397fe0
Reviewed-on: http://gerrit.cloudera.org:8080/4739
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Reviewed-by: Henry Robinson <he...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ee2a06d8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ee2a06d8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ee2a06d8
Branch: refs/heads/hadoop-next
Commit: ee2a06d827a94e659e7d92fe027eeaab6a9ac680
Parents: 3f5380d
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Thu Oct 13 13:24:43 2016 -0700
Committer: Tim Armstrong <ta...@cloudera.com>
Committed: Tue Oct 18 16:35:58 2016 +0000
----------------------------------------------------------------------
bin/bootstrap_toolchain.py | 2 +-
bin/generate_minidump_collection_testdata.py | 1 -
bin/impala-config.sh | 2 --
bin/start-impala-cluster.py | 7 ++-----
common/thrift/ImpalaInternalService.thrift | 6 ++++++
common/thrift/ImpalaService.thrift | 5 ++++-
common/thrift/generate_metrics.py | 4 ----
common/thrift/metrics.json | 20 --------------------
infra/deploy/deploy.py | 9 ---------
9 files changed, 13 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 6524e82..3f161c8 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -351,5 +351,5 @@ if __name__ == "__main__":
# Download the CDH components if necessary.
if os.getenv("DOWNLOAD_CDH_COMPONENTS", "false") == "true":
- cdh_components = ["hadoop", "hbase", "hive", "llama", "llama-minikdc", "sentry"]
+ cdh_components = ["hadoop", "hbase", "hive", "llama-minikdc", "sentry"]
download_cdh_components(toolchain_root, cdh_components)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/generate_minidump_collection_testdata.py
----------------------------------------------------------------------
diff --git a/bin/generate_minidump_collection_testdata.py b/bin/generate_minidump_collection_testdata.py
index a408e05..350f0e3 100755
--- a/bin/generate_minidump_collection_testdata.py
+++ b/bin/generate_minidump_collection_testdata.py
@@ -50,7 +50,6 @@ options, args = parser.parse_args()
CONFIG_FILE = '''-beeswax_port=21000
-fe_port=21000
-be_port=22000
--llama_callback_port=28000
-hs2_port=21050
-enable_webserver=true
-mem_limit=108232130560
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 2a25248..38bdc69 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -299,7 +299,6 @@ export IMPALA_HADOOP_VERSION=${IMPALA_HADOOP_VERSION:-2.6.0-cdh5.10.0-SNAPSHOT}
export IMPALA_HBASE_VERSION=${IMPALA_HBASE_VERSION:-1.2.0-cdh5.10.0-SNAPSHOT}
export IMPALA_HIVE_VERSION=${IMPALA_HIVE_VERSION:-1.1.0-cdh5.10.0-SNAPSHOT}
export IMPALA_SENTRY_VERSION=${IMPALA_SENTRY_VERSION:-1.5.1-cdh5.10.0-SNAPSHOT}
-export IMPALA_LLAMA_VERSION=${IMPALA_LLAMA_VERSION:-1.0.0-cdh5.10.0-SNAPSHOT}
export IMPALA_PARQUET_VERSION=${IMPALA_PARQUET_VERSION:-1.5.0-cdh5.10.0-SNAPSHOT}
export IMPALA_LLAMA_MINIKDC_VERSION=${IMPALA_LLAMA_MINIKDC_VERSION:-1.0.0}
@@ -339,7 +338,6 @@ HADOOP_CLASSPATH+=":$LZO_JAR_PATH"
export MINI_DFS_BASE_DATA_DIR="$IMPALA_HOME/cdh-${CDH_MAJOR_VERSION}-hdfs-data"
export PATH="$HADOOP_HOME/bin:$PATH"
-export LLAMA_HOME="$CDH_COMPONENTS_HOME/llama-${IMPALA_LLAMA_VERSION}/"
export MINIKDC_HOME="$CDH_COMPONENTS_HOME/llama-minikdc-${IMPALA_LLAMA_MINIKDC_VERSION}"
export SENTRY_HOME="$CDH_COMPONENTS_HOME/sentry-${IMPALA_SENTRY_VERSION}"
export SENTRY_CONF_DIR="$IMPALA_HOME/fe/src/test/resources"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/bin/start-impala-cluster.py
----------------------------------------------------------------------
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index b92fcf1..3ea338a 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -85,8 +85,7 @@ MINI_IMPALA_CLUSTER_PATH = IMPALAD_PATH + " -in-process"
IMPALA_SHELL = os.path.join(IMPALA_HOME, 'bin/impala-shell.sh')
IMPALAD_PORTS = ("-beeswax_port=%d -hs2_port=%d -be_port=%d "
- "-state_store_subscriber_port=%d -webserver_port=%d "
- "-llama_callback_port=%d")
+ "-state_store_subscriber_port=%d -webserver_port=%d")
JVM_ARGS = "-jvm_debug_port=%s -jvm_args=%s"
BE_LOGGING_ARGS = "-log_filename=%s -log_dir=%s -v=%s -logbufsecs=5 -max_log_files=%s"
CLUSTER_WAIT_TIMEOUT_IN_SECONDS = 240
@@ -188,12 +187,10 @@ def build_impalad_port_args(instance_num):
BASE_BE_PORT = 22000
BASE_STATE_STORE_SUBSCRIBER_PORT = 23000
BASE_WEBSERVER_PORT = 25000
- BASE_LLAMA_CALLBACK_PORT = 28000
return IMPALAD_PORTS % (BASE_BEESWAX_PORT + instance_num, BASE_HS2_PORT + instance_num,
BASE_BE_PORT + instance_num,
BASE_STATE_STORE_SUBSCRIBER_PORT + instance_num,
- BASE_WEBSERVER_PORT + instance_num,
- BASE_LLAMA_CALLBACK_PORT + instance_num)
+ BASE_WEBSERVER_PORT + instance_num)
def build_impalad_logging_args(instance_num, service_name):
log_file_path = os.path.join(options.log_dir, "%s.INFO" % service_name)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/ImpalaInternalService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaInternalService.thrift b/common/thrift/ImpalaInternalService.thrift
index 3ee54ae..736de34 100644
--- a/common/thrift/ImpalaInternalService.thrift
+++ b/common/thrift/ImpalaInternalService.thrift
@@ -71,11 +71,14 @@ struct TQueryOptions {
6: optional i64 max_scan_range_length = 0
7: optional i32 num_scanner_threads = 0
+ // TODO: IMPALA-4306: retire at compatibility-breaking version
8: optional i32 max_io_buffers = 0 // Deprecated in 1.1
9: optional bool allow_unsupported_formats = 0
+ // TODO: IMPALA-4306: retire at compatibility-breaking version
10: optional i64 default_order_by_limit = -1 // Deprecated in 1.4
11: optional string debug_action = ""
12: optional i64 mem_limit = 0
+ // TODO: IMPALA-4306: retire at compatibility-breaking version
13: optional bool abort_on_default_limit_exceeded = 0 // Deprecated in 1.4
14: optional CatalogObjects.THdfsCompression compression_codec
15: optional i32 hbase_caching = 0
@@ -89,10 +92,12 @@ struct TQueryOptions {
20: optional string request_pool
// Per-host virtual CPU cores required for query (only relevant with RM).
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
21: optional i16 v_cpu_cores
// Max time in milliseconds the resource broker should wait for
// a resource request to be granted by Llama/Yarn (only relevant with RM).
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
22: optional i64 reservation_request_timeout
// Disables taking advantage of HDFS caching. This has two parts:
@@ -104,6 +109,7 @@ struct TQueryOptions {
24: optional bool disable_outermost_topn = 0
// Override for initial memory reservation size if RM is enabled.
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
25: optional i64 rm_initial_mem = 0
// Time, in s, before a query will be timed out if it is inactive. May not exceed
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/ImpalaService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 129be2d..794c140 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -134,21 +134,24 @@ enum TImpalaQueryOptions {
REQUEST_POOL,
// Per-host virtual CPU cores required for query (only relevant with RM).
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
V_CPU_CORES,
// Max time in milliseconds the resource broker should wait for
// a resource request to be granted by Llama/Yarn (only relevant with RM).
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
RESERVATION_REQUEST_TIMEOUT,
// if true, disables cached reads. This option has no effect if REPLICA_PREFERENCE is
// configured.
- // TODO: Retire in C6
+ // TODO: IMPALA-4306: retire at compatibility-breaking version
DISABLE_CACHED_READS,
// Temporary testing flag
DISABLE_OUTERMOST_TOPN,
// Size of initial memory reservation when RM is enabled
+ // TODO: IMPALA-3271: retire at compatibility-breaking version
RM_INITIAL_MEM,
// Time, in s, before a query will be timed out if it is inactive. May not exceed
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/generate_metrics.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py
index e878a29..f1f57cd 100755
--- a/common/thrift/generate_metrics.py
+++ b/common/thrift/generate_metrics.py
@@ -166,10 +166,6 @@ MDL_BASE = """
"name" : "CATALOGSERVER",
"nameForCrossEntityAggregateMetrics" : "catalogservers"
},
- {
- "name" : "LLAMA",
- "nameForCrossEntityAggregateMetrics" : "llamas"
- }
],
"metricEntityTypeDefinitions" : [
{
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/common/thrift/metrics.json
----------------------------------------------------------------------
diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json
index c61c7fe..0829139 100644
--- a/common/thrift/metrics.json
+++ b/common/thrift/metrics.json
@@ -570,26 +570,6 @@
"key": "impala-server.version"
},
{
- "description": "The number of active connections to the Llama Callback Service.",
- "contexts": [
- "IMPALAD"
- ],
- "label": "Llama Callback Service Active Connections",
- "units": "NONE",
- "kind": "GAUGE",
- "key": "impala.thrift-server.llama-callback.connections-in-use"
- },
- {
- "description": "The total number of connections made to the Llama Callback Service over its lifetime.",
- "contexts": [
- "IMPALAD"
- ],
- "label": "Llama Callback Service Total Connections",
- "units": "UNIT",
- "kind": "COUNTER",
- "key": "impala.thrift-server.llama-callback.total-connections"
- },
- {
"description": "The number of active catalog service connections to this Catalog Server.",
"contexts": [
"CATALOGSERVER"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ee2a06d8/infra/deploy/deploy.py
----------------------------------------------------------------------
diff --git a/infra/deploy/deploy.py b/infra/deploy/deploy.py
index 56e3c46..0463111 100644
--- a/infra/deploy/deploy.py
+++ b/infra/deploy/deploy.py
@@ -314,8 +314,6 @@ def create_new_service(api, cluster, new_name, deps, scratch_dirs, master_host):
new_role_name = "%s-%s-%s" % (new_name, rcg.roleType, md5.hexdigest())
new_service.create_role(new_role_name, rcg.roleType, h.hostId)
print "Created new role %s" % (new_role_name,)
- elif rcg.roleType == "LLAMA":
- continue
else:
md5 = hashlib.md5()
md5.update(master_host.hostId)
@@ -367,19 +365,12 @@ def transform_rcg_config(rcg):
new_config.update(transform_port(rcg.name, full, "beeswax_port"))
new_config.update(transform_port(rcg.name, full, "hs2_port"))
new_config.update(transform_port(rcg.name, full, "impalad_webserver_port"))
- new_config.update(transform_port(rcg.name, full, "llama_callback_port"))
new_config.update(transform_port(rcg.name, full, "state_store_subscriber_port"))
elif rcg.roleType == "CATALOGSERVER":
new_config.update(transform_path(rcg.name, full, "log_dir"))
new_config.update(transform_port(rcg.name, full, "catalog_service_port"))
new_config.update(transform_port(rcg.name, full, "catalogserver_webserver_port"))
- elif rcg.roleType == "LLAMA":
- new_config.update(transform_path(rcg.name, full, "llama_log_dir"))
-
- new_config.update(transform_port(rcg.name, full, "llama_am_server_thrift_admin_address"))
- new_config.update(transform_port(rcg.name, full, "llama_http_port"))
- new_config.update(transform_port(rcg.name, full, "llama_port"))
elif rcg.roleType == "STATESTORE":
new_config.update(transform_path(rcg.name, full, "log_dir"))