You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jr...@apache.org on 2017/03/02 19:02:10 UTC
[1/5] incubator-impala git commit: IMPALA-3401: [DOCS] Remove blocks
of content devoted to CM
Repository: incubator-impala
Updated Branches:
refs/heads/master 1609585dc -> c46379602
IMPALA-3401: [DOCS] Remove blocks of content devoted to CM
Change-Id: I07d56463939fe152c1c6124cc0ad5591337bad6b
Reviewed-on: http://gerrit.cloudera.org:8080/6179
Reviewed-by: Ambreen Kazi <am...@cloudera.com>
Tested-by: Impala Public Jenkins
Reviewed-by: John Russell <jr...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8b417c69
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8b417c69
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8b417c69
Branch: refs/heads/master
Commit: 8b417c692140e831deaf54d066e785bacbd5e68b
Parents: 1609585
Author: John Russell <jr...@cloudera.com>
Authored: Tue Feb 28 13:02:00 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Mar 1 22:49:32 2017 +0000
----------------------------------------------------------------------
docs/impala.ditamap | 2 -
docs/impala_keydefs.ditamap | 1 -
docs/shared/impala_common.xml | 26 --
docs/topics/impala_admin.xml | 1 -
docs/topics/impala_admission.xml | 41 ---
docs/topics/impala_authorization.xml | 55 ---
docs/topics/impala_breakpad.xml | 7 +-
docs/topics/impala_cm_installation.xml | 74 ----
docs/topics/impala_config_options.xml | 33 --
docs/topics/impala_hbase.xml | 5 -
docs/topics/impala_howto_rm.xml | 438 -----------------------
docs/topics/impala_incompatible_changes.xml | 80 -----
docs/topics/impala_isilon.xml | 23 --
docs/topics/impala_ldap.xml | 30 --
docs/topics/impala_logging.xml | 23 --
docs/topics/impala_perf_hdfs_caching.xml | 18 -
docs/topics/impala_processes.xml | 22 --
docs/topics/impala_proxy.xml | 38 --
docs/topics/impala_scalability.xml | 7 +-
docs/topics/impala_ssl.xml | 146 --------
docs/topics/impala_txtfile.xml | 9 -
docs/topics/impala_webui.xml | 5 -
22 files changed, 2 insertions(+), 1082 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/impala.ditamap
----------------------------------------------------------------------
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index 10876ba..41730c4 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -51,7 +51,6 @@ under the License.
<topicref href="topics/impala_schema_design.xml"/>
</topicref>
<topicref audience="standalone" href="topics/impala_install.xml#install">
- <topicref href="topics/impala_cm_installation.xml#cm_installation" audience="hidden"/>
<topicref href="topics/impala_noncm_installation.xml#noncm_installation"/>
<!-- Removed when Hue docs were removed from main library. Might be worth resurrecting someday as general Impala+Hue topic. -->
<!-- <topicref href="impala_hue_xi20559.xml"/> -->
@@ -69,7 +68,6 @@ under the License.
<topicref href="topics/impala_admin.xml">
<topicref audience="standalone" href="topics/impala_admission.xml"/>
<topicref audience="standalone" href="topics/impala_resource_management.xml"/>
- <topicref href="topics/impala_howto_rm.xml"/>
<topicref href="topics/impala_timeouts.xml"/>
<topicref href="topics/impala_proxy.xml"/>
<topicref href="topics/impala_disk_space.xml"/>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/impala_keydefs.ditamap
----------------------------------------------------------------------
diff --git a/docs/impala_keydefs.ditamap b/docs/impala_keydefs.ditamap
index 08e0f4f..b48d6d7 100644
--- a/docs/impala_keydefs.ditamap
+++ b/docs/impala_keydefs.ditamap
@@ -10341,7 +10341,6 @@ https://issues.cloudera.org/secure/IssueNavigator.jspa?reset=true&jqlQuery=p
<keydef href="topics/impala_admin.xml" keys="admin"/>
<keydef href="topics/impala_admission.xml" keys="admission_control"/>
<keydef href="topics/impala_resource_management.xml" keys="resource_management"/>
- <keydef href="topics/impala_howto_rm.xml" keys="howto_impala_rm"/>
<keydef href="topics/impala_timeouts.xml" keys="timeouts"/>
<keydef href="topics/impala_proxy.xml" keys="proxy"/>
<keydef href="topics/impala_disk_space.xml" keys="disk_space"/>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/shared/impala_common.xml
----------------------------------------------------------------------
diff --git a/docs/shared/impala_common.xml b/docs/shared/impala_common.xml
index 4604a33..a6d88c3 100644
--- a/docs/shared/impala_common.xml
+++ b/docs/shared/impala_common.xml
@@ -574,20 +574,6 @@ under the License.
in the <cmdname>impalad</cmdname> and <cmdname>catalogd</cmdname> configuration settings.
</p>
</li>
- <li audience="hidden">
- <p>
- For clusters managed by Cloudera Manager, select the
- <uicontrol>Use HDFS Rules to Map Kerberos Principals to Short Names</uicontrol>
- checkbox to enable the service-wide <codeph>load_auth_to_local_rules</codeph> configuration setting.
- Use the Cloudera Manager setting, <uicontrol>Additional Rules to Map Kerberos Principals to Short Names</uicontrol>,
- to insert mapping rules.
- Then restart the Impala service.
- </p>
- <p>
- See <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/sg_auth_to_local_isolate.html" scope="external" format="html">Using Auth-to-Local Rules to Isolate Cluster Users</xref>
- for general information about this feature.
- </p>
- </li>
</ul>
</p>
@@ -608,12 +594,6 @@ under the License.
Sentry logs all facts that lead up to authorization decisions at the debug level. If you do not understand
why Sentry is denying access, the best way to debug is to temporarily turn on debug logging:
<ul>
- <li audience="hidden">
- In Cloudera Manager, add <codeph>log4j.logger.org.apache.sentry=DEBUG</codeph> to the logging settings
- for your service through the corresponding <uicontrol>Logging Safety Valve</uicontrol> field for the
- Impala, Hive Server 2, or Solr Server services.
- </li>
-
<li>
Add <codeph>log4j.logger.org.apache.sentry=DEBUG</codeph> to the <filepath>log4j.properties</filepath>
file on each host in the cluster, in the appropriate configuration directory for each service.
@@ -3367,12 +3347,6 @@ sudo pip-python install ssl</codeblock>
the connection has been closed.
</note>
- <p id="impala_mr">
- For a detailed information about configuring a cluster to share resources between Impala queries and MapReduce jobs, see
- <xref audience="integrated" href="../topics/admin_howto_multitenancy.xml#howto_multitenancy"/><xref audience="standalone" href="https://www.cloudera.com/documentation/enterprise/latest/topics/admin_howto_multitenancy.html" format="html" scope="external"/>
- and <xref href="../topics/impala_howto_rm.xml#howto_impala_rm"/>.
- </p>
-
<note id="llama_beta" type="warning">
In CDH 5.0.0, the Llama component is in beta. It is intended for evaluation of resource management in test
environments, in combination with Impala and YARN. It is currently not recommended for production
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_admin.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_admin.xml b/docs/topics/impala_admin.xml
index 2b9f032..01ed0da 100644
--- a/docs/topics/impala_admin.xml
+++ b/docs/topics/impala_admin.xml
@@ -73,6 +73,5 @@ under the License.
</ul>
</p>
-<!-- <p conref="../shared/impala_common.xml#common/impala_mr"/> -->
</conbody>
</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_admission.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_admission.xml b/docs/topics/impala_admission.xml
index ea47d89..cceae93 100644
--- a/docs/topics/impala_admission.xml
+++ b/docs/topics/impala_admission.xml
@@ -606,32 +606,6 @@ under the License.
</section>
</conbody>
- <concept id="admission_config_cm" audience="hidden">
-
-<!-- TK: Maybe all this stuff overlaps with admin_impala_admission_control and can be delegated there. -->
-
- <title>Configuring Admission Control Using Cloudera Manager</title>
- <prolog>
- <metadata>
- <data name="Category" value="Cloudera Manager"/>
- </metadata>
- </prolog>
-
- <conbody>
-
- <p>
- In Cloudera Manager, you can configure pools to manage queued Impala queries, and the options for the
- limit on number of concurrent queries and how to handle queries that exceed the limit. For details, see
- <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_managing_resources.html" scope="external" format="html">Managing Resources with Cloudera Manager</xref>.
- </p>
-
- <p audience="hidden"><!-- Hiding link because that subtopic is now hidden. -->
- See <xref href="#admission_examples"/> for a sample setup for admission control under
- Cloudera Manager.
- </p>
- </conbody>
- </concept>
-
<concept id="admission_config_noncm">
<title>Configuring Admission Control Using the Command Line</title>
@@ -694,10 +668,6 @@ impala.admission-control.pool-queue-timeout-ms.<varname>queue_name</varname></ph
<keyword keyref="impala25_full"/> and higher.
</p>
- <p audience="hidden"><!-- Hiding link because that subtopic is now hidden. -->
- See <xref href="#admission_examples/section_etq_qgb_rq"/> for sample configuration files
- for admission control using multiple resource pools, without Cloudera Manager.
- </p>
</conbody>
</concept>
@@ -708,17 +678,6 @@ impala.admission-control.pool-queue-timeout-ms.<varname>queue_name</varname></ph
<conbody>
- <section id="section_fqn_qgb_rq" audience="hidden">
-
- <title>Example Admission Control Configurations Using Cloudera Manager</title>
-
- <p>
- For full instructions about configuring dynamic resource pools through Cloudera Manager, see
- <xref audience="integrated" href="cm_mc_resource_pools.xml#xd_583c10bfdbd326ba--43d5fd93-1410993f8c2--7ff2"/><xref audience="standalone" href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_resource_pools.html" scope="external" format="html"/>.
- </p>
-
- </section>
-
<section id="section_etq_qgb_rq">
<title>Example Admission Control Configurations Using Configuration Files</title>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_authorization.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_authorization.xml b/docs/topics/impala_authorization.xml
index 7890d71..7a360fa 100644
--- a/docs/topics/impala_authorization.xml
+++ b/docs/topics/impala_authorization.xml
@@ -223,16 +223,6 @@ under the License.
</p>
<ul>
- <li audience="hidden">
- <p>
- In an environment managed by Cloudera Manager, the server name is specified through
- <menucascade><uicontrol>Impala (Service-Wide)</uicontrol><uicontrol>Category</uicontrol><uicontrol>Advanced</uicontrol><uicontrol>Sentry Service</uicontrol></menucascade> and
- <menucascade><uicontrol>Hive</uicontrol><uicontrol>Service-Wide</uicontrol><uicontrol>Advanced</uicontrol><uicontrol>Sentry Service</uicontrol></menucascade>.
- The values must be the same for both, so that Impala and Hive can share the privilege rules. Restart
- the Impala and Hive services after setting or changing this value.
- </p>
- </li>
-
<li>
<p>
Specify the <codeph>server1</codeph> value for the <codeph>sentry.hive.server</codeph> property in the
@@ -283,20 +273,6 @@ report_generator = server=server1->db=reporting_db->table=*->action=SEL
<codeph>REVOKE</codeph> statements in <keyword keyref="impala20_full"/>.)
</p>
- <p audience="hidden">
- Hive already had <codeph>GRANT</codeph> and <codeph>REVOKE</codeph> statements prior to CDH 5.1, but those
- statements were not production-ready. CDH 5.1 is the first release where those statements use the Sentry
- framework and are considered GA level. If you used the Hive <codeph>GRANT</codeph> and
- <codeph>REVOKE</codeph> statements prior to CDH 5.1, you must set up these privileges with the CDH 5.1
- versions of <codeph>GRANT</codeph> and <codeph>REVOKE</codeph> to take advantage of Sentry authorization.
- </p>
-
- <p audience="hidden">
- For information about using the updated Hive <codeph>GRANT</codeph> and <codeph>REVOKE</codeph> statements,
- see
- <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_sg_sentry_service.html" scope="external" format="html">Sentry
- service</xref> topic in the <cite>CDH 5 Security Guide</cite>.
- </p>
</conbody>
</concept>
@@ -1584,37 +1560,6 @@ column-level security was by creating views that referenced particular sets of c
</conbody>
</concept>
- <concept id="sentry_cm" audience="hidden">
-
- <title>Managing Sentry for Impala through Cloudera Manager</title>
-
- <conbody>
-
- <p>
- To enable the Sentry service for Impala and Hive, set <uicontrol>Hive/Impala > Service-Wide > Sentry
- Service</uicontrol> parameter to the Sentry service. Then restart Impala and Hive. Simply adding Sentry
- service as a dependency and restarting enables Impala and Hive to use the Sentry service.
- </p>
-
- <p>
- To set the server name to use when granting server level privileges, set the <uicontrol>Hive >
- Service-Wide > Advanced > Server Name for Sentry Authorization</uicontrol> parameter. When using
- Sentry with the Hive Metastore, you can specify the list of users that are allowed to bypass Sentry
- Authorization in Hive Metastore using <uicontrol>Hive > Service-Wide > Security > Bypass Sentry
- Authorization Users</uicontrol>. These are usually service users that already ensure all activity has been
- authorized.
- </p>
-
- <note>
- The <uicontrol>Hive/Impala > Service-Wide > Policy File Based Sentry</uicontrol> tab contains
- parameters only relevant to configuring Sentry using policy files. In particular, make sure that
- <uicontrol>Enable Sentry Authorization using Policy Files</uicontrol> parameter is unchecked when using the
- Sentry service. Cloudera Manager throws a validation error if you attempt to configure the Sentry service
- and policy file at the same time.
- </note>
- </conbody>
- </concept>
-
<concept id="sec_ex_default">
<title>The DEFAULT Database in a Secure Deployment</title>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_breakpad.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_breakpad.xml b/docs/topics/impala_breakpad.xml
index e5e8cc6..5d755fd 100644
--- a/docs/topics/impala_breakpad.xml
+++ b/docs/topics/impala_breakpad.xml
@@ -85,14 +85,9 @@ under the License.
<!-- Location stated in IMPALA-3581; overridden by different location from IMPALA-2686?
<filepath><varname>log_directory</varname>/minidumps/<varname>daemon_name</varname></filepath> -->
<ul>
- <li audience="hidden">
- <p>
- Clusters managed by Cloudera Manager: <filepath>/var/log/impala-minidumps/<varname>daemon_name</varname></filepath>
- </p>
- </li>
<li>
<p>
- Clusters not managed by Cloudera Manager:
+ Clusters not managed by cluster management software:
<filepath><varname>impala_log_dir</varname>/<varname>daemon_name</varname>/minidumps/<varname>daemon_name</varname></filepath>
</p>
</li>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_cm_installation.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_cm_installation.xml b/docs/topics/impala_cm_installation.xml
deleted file mode 100644
index f1a3b04..0000000
--- a/docs/topics/impala_cm_installation.xml
+++ /dev/null
@@ -1,74 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
-<concept id="cm_installation" audience="hidden">
-
- <title>Installing Impala with Cloudera Manager</title>
- <prolog>
- <metadata>
- <data name="Category" value="Impala"/>
- <data name="Category" value="Installing"/>
- <data name="Category" value="Cloudera Manager"/>
- <data name="Category" value="Administrators"/>
- </metadata>
- </prolog>
-
- <conbody>
-
- <p>
- Before installing Impala through the Cloudera Manager interface, make sure all applicable nodes have the
- appropriate hardware configuration and levels of operating system and CDH. See
- <xref href="impala_prereqs.xml#prereqs"/> for details.
- </p>
-
- <note rev="1.2.0">
- <p rev="1.2.0">
- To install the latest Impala under CDH 4, upgrade Cloudera Manager to 4.8 or higher. Cloudera Manager 4.8 is
- the first release that can manage the Impala catalog service introduced in Impala 1.2. Cloudera Manager 4.8
- requires this service to be present, so if you upgrade to Cloudera Manager 4.8, also upgrade Impala to the
- most recent version at the same time.
-<!-- Not so relevant now for 1.1.1, but maybe someday we'll capture all this history in a compatibility grid.
- Upgrade to Cloudera Manager 4.6.2 or higher to enable Cloudera Manager to
- handle access control for the Impala web UI, available by default through
- port 25000 on each Impala host.
- -->
- </p>
- </note>
-
- <p>
- For information on installing Impala in a Cloudera Manager-managed environment, see
- <xref audience="integrated" href="cm_ig_install_impala.xml"/><xref audience="standalone" href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_ig_install_impala.html" scope="external" format="html">Installing Impala</xref>.
- </p>
-
- <p>
- Managing your Impala installation through Cloudera Manager has a number of advantages. For example, when you
- make configuration changes to CDH components using Cloudera Manager, it automatically applies changes to the
- copies of configuration files, such as <codeph>hive-site.xml</codeph>, that Impala keeps under
- <filepath>/etc/impala/conf</filepath>. It also sets up the Hive Metastore service that is required for
- Impala running under CDH 4.1.
- </p>
-
- <p>
- In some cases, depending on the level of Impala, CDH, and Cloudera Manager, you might need to add particular
- component configuration details in some of the free-form option fields on the Impala configuration pages
- within Cloudera Manager. <ph conref="../shared/impala_common.xml#common/safety_valve"/>
- </p>
- </conbody>
-</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_config_options.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_config_options.xml b/docs/topics/impala_config_options.xml
index 3260b75..3c1ae1f 100644
--- a/docs/topics/impala_config_options.xml
+++ b/docs/topics/impala_config_options.xml
@@ -66,35 +66,6 @@ under the License.
</conbody>
- <concept id="config_options_cm" audience="hidden">
-
- <title>Configuring Impala Startup Options through Cloudera Manager</title>
-
- <conbody>
-
- <p>
- If you manage your cluster through Cloudera Manager, configure the settings for all the
- Impala-related daemons by navigating to this page:
- <menucascade><uicontrol>Clusters</uicontrol><uicontrol>Impala</uicontrol><uicontrol>Configuration</uicontrol><uicontrol>View
- and Edit</uicontrol></menucascade>. See the Cloudera Manager documentation for
- <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_impala_service.html" scope="external" format="html">instructions
- about how to configure Impala through Cloudera Manager</xref>.
- </p>
-
- <p>
- If the Cloudera Manager interface does not yet have a form field for a newly added
- option, or if you need to use special options for debugging and troubleshooting, the
- <uicontrol>Advanced</uicontrol> option page for each daemon includes one or more fields
- where you can enter option names directly.
- <ph conref="../shared/impala_common.xml#common/safety_valve"/> There is also a free-form
- field for query options, on the top-level <uicontrol>Impala Daemon</uicontrol> options
- page.
- </p>
-
- </conbody>
-
- </concept>
-
<concept id="config_options_noncm">
<title>Configuring Impala Startup Options through the Command Line</title>
@@ -237,10 +208,6 @@ Starting Impala Catalog Server: [ OK ]</codeblock>
to:
</p>
<codeblock>export ENABLE_CORE_DUMPS=${ENABLE_COREDUMPS:-true}</codeblock>
- <p audience="hidden">
- On systems managed by Cloudera Manager, enable the <uicontrol>Enable Core
- Dump</uicontrol> setting for the Impala service.
- </p>
<note conref="../shared/impala_common.xml#common/core_dump_considerations"/>
</li>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_hbase.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_hbase.xml b/docs/topics/impala_hbase.xml
index 0068e48..0b3754b 100644
--- a/docs/topics/impala_hbase.xml
+++ b/docs/topics/impala_hbase.xml
@@ -156,11 +156,6 @@ under the License.
</property>
</codeblock>
- <p audience="hidden">
- Currently, Cloudera Manager does not have an Impala-only override for HBase settings, so any HBase
- configuration change you make through Cloudera Manager would take affect for all HBase applications.
- Therefore, this change is not recommended on systems managed by Cloudera Manager.
- </p>
</conbody>
</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_howto_rm.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_howto_rm.xml b/docs/topics/impala_howto_rm.xml
deleted file mode 100644
index 9e4361a..0000000
--- a/docs/topics/impala_howto_rm.xml
+++ /dev/null
@@ -1,438 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
-<concept id="howto_impala_rm">
-
- <title>How to Configure Resource Management for Impala</title>
- <prolog>
- <metadata>
- <data name="Category" value="Impala"/>
- <data name="Category" value="Admission Control"/>
- <data name="Category" value="Resource Management"/>
- </metadata>
- </prolog>
- <conbody>
- <p>Impala includes features that balance and maximize resources in your CDH cluster. This topic describes
- how you can enhance a CDH cluster using Impala to improve efficiency.</p>
- <p outputclass="toc inpage">A typical deployment uses the following.</p>
- <ul audience="HTML">
- <li audience="HTML">Creating Static Service Pools</li>
- <li audience="HTML">Using Admission Control <ul>
- <li audience="HTML">Setting Per-query Memory Limits</li>
- <li audience="HTML">Creating Dynamic Resource Pools</li>
- </ul></li>
-
- </ul>
- </conbody>
- <concept id="static_service_pools">
- <title>Creating Static Service Pools</title>
- <conbody>
- <p>Use Static Service Pools to allocate dedicated resources for Impala and other services to allow
- for predictable resource availability. </p>
- <p>Static service pools isolate services from one another, so that high load on one service has
- bounded impact on other services. You can use Cloudera Manager to configure static service pools
- that control memory, CPU and Disk I/O.</p>
- <p >The following screenshot shows a sample configuration for Static Service Pools in
- Cloudera Manager:</p>
- <draft-comment author="ddawson">
- <p>Need accurate numbers - or can we remove services other than HDFS, Impala, and YARN? Matt
- Jacobs is going to run these numbers by someone in the field.</p>
- </draft-comment>
- <p>
- <image href="../images/howto_static_server_pools_config.png" placement="break"
- id="PDF_33" align="center" scale="33" audience="HTML"/>
- <image href="../images/howto_static_server_pools_config.png" placement="break"
- id="HTML_SCALEFIT" align="center" scalefit="yes" audience="PDF"/>
- </p>
- <p>
- <ul id="ul_tkw_4rs_pw">
- <li >
- <p >HDFS always needs to have a minimum of 5-10% of the resources.</p>
- </li>
- <li >
- Generally, YARN and Impala split the rest of the resources.
-
- <ul id="ul_ukw_4rs_pw">
- <li >
- <p >For mostly batch workloads, you might allocate YARN 60%, Impala 30%, and HDFS
- 10%. </p>
- </li>
- <li >
- <p >For mostly ad hoc query workloads, you might allocate Impala 60%, YARN 30%, and
- HDFS 10%.</p>
- </li>
- </ul></li>
- </ul>
- </p>
- </conbody>
- </concept>
- <concept id="enable_admission_control">
- <title>Using Admission Control</title>
- <conbody>
- <p>Within the constraints of the static service pool, you can further subdivide Impala's
- resources using Admission Control. You configure Impala Admission Control pools in the Cloudera
- Manager Dynamic Resource Pools page.</p>
- <p>You use Admission Control to divide usage between Dynamic Resource Pools in multitenant use
- cases. Allocating resources judiciously allows your most important queries to run faster and
- more reliably.</p>
- <p>
- <note>In this context, Impala Dynamic Resource Pools are different than the default YARN Dynamic
- Resource Pools. You can turn on Dynamic Resource Pools that are exclusively for use by
- Impala.</note>
- </p>
- <p>Admission Control is enabled by default.</p>
- <p>A Dynamic Resource Pool has the following properties:<ul id="ul_blk_jjg_sw">
- <li><b>Max Running Queries</b>: Maximum number of concerrently executing queries in the pool
- before incoming queries are queued.</li>
- <li><b>Max Memory Resources</b>: Maximum memory used by queries in the pool before incoming
- queries are queued. This value is used at the time of admission and is not enforced at query
- runtime.</li>
- <li><b>Default Query Memory Limit</b>: Defines the maximum amount of memory a query can
- allocate on each node. This is enforced at runtime. If the query attempts to use more memory,
- it is forced to spill, if possible. Otherwise, it is cancelled. The total memory that can be
- used by a query is the <codeph>MEM_LIMIT</codeph> times the number of nodes.</li>
- <li><b>Max Queued Queries</b>: Maximum number of queries that can be queued in the pool before
- additional queries are rejected.</li>
- <li><b>Queue Timeout</b>: Specifies how long queries can wait in the queue before they are
- cancelled with a timeout error.</li>
- </ul></p>
- </conbody>
- </concept>
- <concept id="set_per_query_memory_limits">
- <title>Setting Per-query Memory Limits</title>
- <conbody>
- <p>Use per-query memory limits to prevent queries from consuming excessive memory resources that
- impact other queries. <ph rev="upstream">Cloudera</ph> recommends that you set the query memory limits whenever
- possible.</p>
- <p>If you set the <b>Pool Max Mem Resources</b> for a resource pool, Impala attempts to throttle
- queries if there is not enough memory to run them within the specified resources.</p>
- <p>Only use admission control with maximum memory resources if you can ensure there are query
- memory limits. Set the pool <b>Default Query Memory Limit</b> to be certain. You can override
- this setting with the <codeph>query</codeph> option, if necessary.</p>
- <p>Typically, you set query memory limits using the <codeph>set MEM_LIMIT=Xg;</codeph> query
- option. When you find the right value for your business case, memory-based admission control
- works well. The potential downside is that queries that attempt to use more memory might perform
- poorly or even be cancelled.</p>
- <p>To find a reasonable default query memory limit:<ol id="ol_ydt_xhy_pw">
- <li>Run the workload.</li>
- <li>In Cloudera Manager, go to <menucascade>
- <uicontrol>Impala</uicontrol>
- <uicontrol>Queries</uicontrol>
- </menucascade>.</li>
- <li>Click <uicontrol>Select Attributes</uicontrol>.</li>
- <li>Select <uicontrol>Per Node Peak Memory Usage</uicontrol> and click
- <uicontrol>Update</uicontrol>.</li>
- <li>Allow the system time to gather information, then click the <uicontrol>Show
- Histogram</uicontrol> icon to see the results.<image placement="break"
- href="../images/howto_show_histogram.png" align="center" id="image_hmv_xky_pw"/></li>
- <li>Use the histogram to find a value that accounts for most queries. Queries that require more
- resources than this limit should explicitly set the memory limit to ensure they can run to
- completion.<draft-comment author="ddawson">This chart uses bad sample data - we will change
- the chart when we have real numbers from the sample use case.</draft-comment><image
- placement="break" href="../images/howto_per_node_peak_memory_usage.png" align="center"
- id="image_ehn_hly_pw" scalefit="yes"/></li>
- </ol></p>
- </conbody>
- </concept>
- <concept id="concept_en4_3sy_pw">
- <title>Creating Dynamic Resource Pools</title>
- <conbody>
- <p>A dynamic resource pool is a named configuration of resources and a policy for scheduling the
- resources among Impala queries running in the pool. Dynamic resource pools allow you to schedule
- and allocate resources to Impala queries based on a user's access to specific pools and the
- resources available to those pools. </p>
- <p>This example creates both production and development resource pools or queues. It assumes you
- have 3 worker nodes with 24GiB of RAM each for an aggregate memory of 72000MiB. This pool
- configuration allocates the Production queue twice the memory resources of the Development
- queue, and a higher number of concurrent queries.</p>
- <p>To create a Production dynamic resource pool for Impala:</p>
- <ol>
- <li>In Cloudera Manager, select <menucascade>
- <uicontrol>Clusters</uicontrol>
- <uicontrol>Dynamic Resource Pool Configuration</uicontrol>
- </menucascade>.</li>
- <li>Click the <uicontrol>Impala Admission Control</uicontrol> tab.</li>
- <li>Click <b>Create Resource Pool</b>.</li>
- <li>Specify a name and resource limits for the Production pool:<ul id="ul_rjt_wqv_2v">
- <li>In the <b>Resource Pool Name</b> field, enter <userinput>Production</userinput>.</li>
- <li>In the <uicontrol>Max Memory</uicontrol> field, enter <userinput>48000</userinput>.</li>
- <li>In the <uicontrol>Default Query Memory Limit</uicontrol> field, enter
- <userinput>1600</userinput>.</li>
- <li>In the <uicontrol>Max Running Queries</uicontrol> field, enter
- <userinput>10</userinput>.</li>
- <li>In the <uicontrol>Max Queued Queries</uicontrol> field, enter
- <userinput>200</userinput>.</li>
- </ul></li>
- <li>Click <uicontrol>Create</uicontrol>.</li>
- <li>Click <uicontrol>Refresh Dynamic Resource Pools</uicontrol>.</li>
- </ol>
- <p>The Production queue runs up to 10 queries at once. If the total memory requested
- by these queries exceeds 48000 MiB, it holds the next query in the queue until the memory is
- released. It also prevents a query from running if it needs more memory than is currently
- available. Admission Control holds the next query if either Max Running Queries is reached, or
- the pool Max Memory limit is reached.</p>
- <p>Here, Max Memory resources and Default Query Memory Limit throttle throughput to 10 queries,
- so setting Max Running Queries might not be necessary, though it does not hurt to do so. Most
- users set Max Running Queries when they cannot pick good numbers for memory. Since users can
- override the query option <varname>mem_limit</varname>, setting the Max Running Queries property
- might make sense.</p>
- <p>To create a Development dynamic resource pool for Impala:</p>
-
- <ol>
- <li>In Cloudera Manager, select <menucascade>
- <uicontrol>Clusters</uicontrol>
- <uicontrol>Dynamic Resource Pool Configuration</uicontrol>
- </menucascade>.</li>
- <li>Click the <uicontrol>Impala Admission Control</uicontrol> tab.</li>
- <li>Click <b>Create Resource Pool</b>.</li>
- <li>Specify a name and resource limits for the Development pool:<ul id="ul_j42_q3z_pw">
- <li>In the <b>Resource Pool Name</b> field, enter <userinput>Development</userinput>.</li>
- <li>In the <uicontrol>Max Memory</uicontrol> field, enter <userinput>24000</userinput>.</li>
- <li>In the <uicontrol>Default Query Memory Limit</uicontrol> field, enter
- <userinput>8000</userinput>.</li>
- <li>In the <uicontrol>Max Running Queries</uicontrol> field, enter 1.</li>
- <li>In the <uicontrol>Max Queued Queries</uicontrol> field, enter 100.</li>
- </ul></li>
- <li>Click <uicontrol>Create</uicontrol>.</li>
- <li>Click <uicontrol>Refresh Dynamic Resource Pools</uicontrol>.<p>The Development queue runs
- one query at a time. If the total memory required by the query exceeds 24000 MiB, it holds the
- query until memory is released.</p></li>
- </ol>
- </conbody>
- <concept id="setting_placement_rules">
- <title>Understanding Placement Rules</title>
- <conbody>
- <p>Placement rules determine how queries are mapped to resource pools. The standard settings are
- to use a specified pool when specified; otherwise, use the default pool.</p>
- <p>For example, you can use the SET statement to select the pool in which to run a
- query.<codeblock>SET REQUEST_POOL=Production;</codeblock></p>
- <p>If you do not use a <codeph>SET</codeph> statement, queries are run in the default pool.</p>
- </conbody>
- </concept>
- <concept id="setting_access_control_on_pools">
- <title>Setting Access Control on Pools</title>
- <conbody>
- <p>You can specify that only cetain users and groups are allowed to use the pools you define.</p>
- <p>To create a Development dynamic resource pool for Impala:</p>
- <ol>
- <li>In Cloudera Manager, select <menucascade>
- <uicontrol>Clusters</uicontrol>
- <uicontrol>Dynamic Resource Pool Configuration</uicontrol>
- </menucascade>.</li>
- <li>Click the <uicontrol>Impala Admission Control</uicontrol> tab.</li>
- <li>Click the <uicontrol>Edit</uicontrol> button for the Production pool.</li>
- <li>Click the Submission Access Control tab.</li>
- <li>Select <uicontrol>Allow these users and groups to submit to this pool</uicontrol>.</li>
- <li>Enter a comma-separated list of users who can use the pool.
- <image placement="break"
- href="../images/howto_access_control.png" align="center" scalefit="yes"/>
- </li>
-
- <li>Click <uicontrol>Save</uicontrol>.</li>
- </ol>
- </conbody>
- </concept>
- </concept>
-<concept id="impala_resource_management_example">
- <title>Impala Resource Management Example</title>
- <conbody>
- <p>Anne Chang is administrator for an enterprise data hub that runs a number of workloads,
- including Impala. </p>
- <p>Anne has a 20-node cluster that uses Cloudera Manager static partitioning. Because of the
- heavy Impala workload, Anne needs to make sure Impala gets enough resources. While the best
- configuration values might not be known in advance, she decides to start by allocating 50% of
- resources to Impala. Each node has 128 GiB dedicated to each impalad. Impala has 2560 GiB in
- aggregate that can be shared across the resource pools she creates.</p>
- <p>Next, Anne studies the workload in more detail. After some research, she might choose to
- revisit these initial values for static partitioning. </p>
- <p>To figure out how to further allocate Impala\u2019s resources, Anne needs to consider the workloads
- and users, and determine their requirements. There are a few main sources of Impala queries: <ul
- id="ul_ml3_sf2_5w">
- <li>Large reporting queries executed by an external process/tool. These are critical business
- intelligence queries that are important for business decisions. It is important that they get
- the resources they need to run. There typically are not many of these queries at a given
- time.</li>
- <li>Frequent, small queries generated by a web UI. These queries scan a limited amount of data
- and do not require expensive joins or aggregations. These queries are important, but not as
- critical, perhaps the client tries resending the query or the end user refreshes the
- page.</li>
- <li>Occasionally, expert users might run ad-hoc queries. The queries can vary significantly in
- their resource requirements. While Anne wants a good experience for these users, it is hard to
- control what they do (for example, submitting inefficient or incorrect queries by mistake).
- Anne restricts these queries by default and tells users to reach out to her if they need more
- resources. </li>
- </ul></p>
- <p>To set up admission control for this workload, Anne first runs the workloads independently, so
- that she can observe the workload\u2019s resource usage in Cloudera Manager. If they could not easily
- be run manually, but had been run in the past, Anne uses the history information from Cloudera
- Manager. It can be helpful to use other search criteria (for example, <i>user</i>) to isolate
- queries by workload. Anne uses the Cloudera Manager chart for Per-Node Peak Memory usage to
- identify the maximum memory requirements for the queries. </p>
- <p>From this data, Anne observes the following about the queries in the groups above:<ul
- id="ul_amq_ng2_5w">
- <li> Large reporting queries use up to 32 GiB per node. There are typically 1 or 2 queries
- running at a time. On one occasion, she observed that 3 of these queries were running
- concurrently. Queries can take 3 minutes to complete.</li>
- <li>Web UI-generated queries use between 100 MiB per node to usually less than 4 GiB per node
- of memory, but occasionally as much as 10 GiB per node. Queries take, on average, 5 seconds,
- and there can be as many as 140 incoming queries per minute.</li>
- <li>Anne has little data on ad hoc queries, but some are trivial (approximately 100 MiB per
- node), others join several tables (requiring a few GiB per node), and one user submitted a
- huge cross join of all tables that used all system resources (that was likely a mistake).</li>
- </ul></p>
- <p>Based on these observations, Anne creates the admission control configuration with the
- following pools: </p>
- <section id="section_yjc_h32_5w">
- <title>XL_Reporting</title>
- <p>
- <table frame="all" rowsep="1" colsep="1" id="XL_Reporting_Table">
- <tgroup cols="2">
- <colspec colname="c1" colnum="1" colwidth="1.0*"/>
- <colspec colname="c2" colnum="2" colwidth="1.0*"/>
- <thead>
- <row>
- <entry>Property</entry>
- <entry>Value</entry>
- </row>
- </thead>
- <tbody>
- <row>
- <entry>Max Memory</entry>
- <entry>1280 GiB</entry>
- </row>
- <row>
- <entry>Default Query Memory Limit</entry>
- <entry>32 GiB</entry>
- </row>
- <row>
- <entry>Max Running Queries</entry>
- <entry>2</entry>
- </row>
- <row>
- <entry>Queue Timeout</entry>
- <entry>5 minutes</entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- </p>
- <p>This pool is for large reporting queries. In order to support running 2 queries at a time,
- the pool memory resources are set to 1280 GiB (aggregate cluster memory). This is for 2
- queries, each with 32 GiB per node, across 20 nodes. Anne sets the pool\u2019s Default Query Memory
- Limit to 32 GiB so that no query uses more than 32 GiB on any given node. She sets Max Running
- Queries to 2 (though it is not necessary she do so). She increases the pool\u2019s queue timeout to
- 5 minutes in case a third query comes in and has to wait. She does not expect more than 3
- concurrent queries, and she does not want them to wait that long anyway, so she does not
- increase the queue timeout. If the workload increases in the future, she might choose to adjust
- the configuration or buy more hardware. </p>
- </section>
- <section id="section_xm3_j32_5w"><title>HighThroughput_UI</title>
- <p>
- <table frame="all" rowsep="1" colsep="1" id="High_Throughput_UI_Table">
- <tgroup cols="2">
- <colspec colname="c1" colnum="1" colwidth="1.0*"/>
- <colspec colname="c2" colnum="2" colwidth="1.0*"/>
- <thead>
- <row>
- <entry>Property</entry>
- <entry>Value</entry>
- </row>
- </thead>
- <tbody>
- <row>
- <entry>Max Memory</entry>
- <entry>960 GiB (inferred)</entry>
- </row>
- <row>
- <entry>Default Query Memory Limit</entry>
- <entry>4 GiB</entry>
- </row>
- <row>
- <entry>Max Running Queries</entry>
- <entry>12</entry>
- </row>
- <row>
- <entry>Queue Timeout</entry>
- <entry>5 minutes</entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- </p>
- <p>This pool is used for the small, high throughput queries generated by the web tool. Anne sets
- the Default Query Memory Limit to 4 GiB per node, and sets Max Running Queries to 12. This
- implies a maximum amount of memory per node used by the queries in this pool: 48 GiB per node
- (12 queries * 4 GiB per node memory limit).</p><p>Notice that Anne does not set the pool memory resources, but does set the pool\u2019s Default Query
- Memory Limit. This is intentional: admission control processes queries faster when a pool uses
- the Max Running Queries limit instead of the peak memory resources.</p><p>This should be enough memory for most queries, since only a few go over 4 GiB per node. For those
- that do require more memory, they can probably still complete with less memory (spilling if
- necessary). If, on occasion, a query cannot run with this much memory and it fails, Anne might
- reconsider this configuration later, or perhaps she does not need to worry about a few rare
- failures from this web UI.</p><p>With regard to throughput, since these queries take around 5 seconds and she is allowing 12
- concurrent queries, the pool should be able to handle approximately 144 queries per minute,
- which is enough for the peak maximum expected of 140 queries per minute. In case there is a
- large burst of queries, Anne wants them to queue. The default maximum size of the queue is
- already 200, which should be more than large enough. Anne does not need to change it.</p></section>
- <section id="section_asm_yj2_5w"><title>Default</title>
- <p>
- <table frame="all" rowsep="1" colsep="1" id="default_table">
- <tgroup cols="2">
- <colspec colname="c1" colnum="1" colwidth="1.0*"/>
- <colspec colname="c2" colnum="2" colwidth="1.0*"/>
- <thead>
- <row>
- <entry>Property</entry>
- <entry>Value</entry>
- </row>
- </thead>
- <tbody>
- <row>
- <entry>Max Memory</entry>
- <entry>320 GiB</entry>
- </row>
- <row>
- <entry>Default Query Memory Limit</entry>
- <entry>4 GiB</entry>
- </row>
- <row>
- <entry>Max Running Queries</entry>
- <entry>Unlimited</entry>
- </row>
- <row>
- <entry>Queue Timeout</entry>
- <entry>60 Seconds</entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- </p><p>The default pool (which already exists) is a catch all for ad-hoc queries. Anne wants to use the
- remaining memory not used by the first two pools, 16 GiB per node (XL_Reporting uses 64 GiB per
- node, High_Throughput_UI uses 48 GiB per node). For the other pools to get the resources they
- expect, she must still set the Max Memory resources and the Default Query Memory Limit. She
- sets the Max Memory resources to 320 GiB (16 * 20). She sets the Default Query Memory Limit to
- 4 GiB per node for now. That is somewhat arbitrary, but satisfies some of the ad hoc queries
- she observed. If someone writes a bad query by mistake, she does not actually want it using all
- the system resources. If a user has a large query to submit, an expert user can override the
- Default Query Memory Limit (up to 16 GiB per node, since that is bound by the pool Max Memory
- resources). If that is still insufficient for this user\u2019s workload, the user should work with
- Anne to adjust the settings and perhaps create a dedicated pool for the workload.</p></section>
- </conbody>
-</concept>
-</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_incompatible_changes.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_incompatible_changes.xml b/docs/topics/impala_incompatible_changes.xml
index a25f327..897d973 100644
--- a/docs/topics/impala_incompatible_changes.xml
+++ b/docs/topics/impala_incompatible_changes.xml
@@ -1226,8 +1226,6 @@ select * from `cross`;</codeblock>
to Impala 1.2.x in general.
</p>
- <p audience="hidden" conref="../shared/impala_common.xml#common/cm48_upgrade"/>
-
<!-- <note conref="common.xml#common/cdh4_cdh5_upgrade"/> -->
</conbody>
</concept>
@@ -1268,9 +1266,6 @@ select * from `cross`;</codeblock>
<li/>
</ul>
- <p audience="hidden" conref="../shared/impala_common.xml#common/cm48_upgrade"/>
-
-<!-- <note conref="common.xml#common/cdh4_cdh5_upgrade"/> -->
</conbody>
</concept>
@@ -1405,83 +1400,8 @@ ALTER TABLE <varname>table_name</varname> SET FILEFORMAT
<codeph>impala-lzo-cdh4</codeph> to the latest level. See <xref href="impala_txtfile.xml#lzo"/> for
details.
</li>
-
- <li>
- Cloudera Manager 4.5.2 and higher only supports Impala 1.0 and higher, and vice versa. If you upgrade to
- Impala 1.0 or higher managed by Cloudera Manager, you must also upgrade Cloudera Manager to version 4.5.2
- or higher. If you upgrade from an earlier version of Cloudera Manager, and were using Impala, you must
- also upgrade Impala to version 1.0 or higher. The beta versions of Impala are no longer supported as of
- the release of Impala 1.0.
- </li>
- </ul>
- </conbody>
- </concept>
-
- <concept id="incompatible_changes_07" audience="hidden">
-
- <title>Incompatible Change Introduced in Version 0.7 of the Impala Beta Release</title>
-
- <conbody>
-
- <ul>
- <li>
- The defaults for the <codeph>-nn</codeph> and <codeph>-nn_port</codeph> flags have changed and are now
- read from <codeph>core-site.xml</codeph>. Impala prints the values of <codeph>-nn</codeph> and
- <codeph>-nn_port</codeph> to the log when it starts. The ability to set <codeph>-nn</codeph> and
- <codeph>-nn_port</codeph> on the command line is deprecated in 0.7 and may be removed in Impala 0.8.
- </li>
- </ul>
- </conbody>
- </concept>
-
- <concept id="incompatible_changes_06" audience="hidden">
-
- <title>Incompatible Change Introduced in Version 0.6 of the Impala Beta Release</title>
-
- <conbody>
-
- <ul>
- <li>
- Cloudera Manager 4.5 supports only version 0.6 of the Impala Beta Release. It does not support
- the earlier beta versions. If you upgrade your Cloudera Manager installation, you must also upgrade
- Impala to beta version 0.6. If you upgrade Impala to beta version 0.6, you must upgrade Cloudera Manager
- to 4.5.
- </li>
- </ul>
- </conbody>
- </concept>
-
- <concept id="incompatible_changes_04" audience="hidden">
-
- <title>Incompatible Change Introduced in Version 0.4 of the Impala Beta Release</title>
-
- <conbody>
-
- <ul>
- <li>
- Cloudera Manager 4.1.3 supports only version 0.4 of the Impala Beta Release. It does not support
- the earlier beta versions. If you upgrade your Cloudera Manager installation, you must also upgrade
- Impala to beta version 0.4. If you upgrade Impala to beta version 0.4, you must upgrade Cloudera Manager
- to 4.1.3.
- </li>
</ul>
</conbody>
</concept>
- <concept id="incompatible_changes_03" audience="hidden">
-
- <title>Incompatible Change Introduced in Version 0.3 of the Impala Beta Release</title>
-
- <conbody>
-
- <ul>
- <li>
- Cloudera Manager 4.1.2 supports only version 0.3 of the Impala Beta Release. It does not support
- the earlier beta versions. If you upgrade your Cloudera Manager installation, you must also upgrade
- Impala to beta version 0.3. If you upgrade Impala to beta version 0.3, you must upgrade Cloudera Manager
- to 4.1.2.
- </li>
- </ul>
- </conbody>
- </concept>
</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_isilon.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_isilon.xml b/docs/topics/impala_isilon.xml
index bf93b39..12e6343 100644
--- a/docs/topics/impala_isilon.xml
+++ b/docs/topics/impala_isilon.xml
@@ -102,30 +102,7 @@ CREATE TABLE d1.t2 (a TINYINT, b BOOLEAN);
for the <cmdname>impalad</cmdname> daemon on clusters not using Cloudera Manager.
</p>
- <p audience="hidden">
-<!--
- For information about tasks performed on
- Isilon OneFS, see the information hub for Cloudera on the EMC Community Network:
- <xref href="https://community.emc.com/docs/DOC-39522" format="html" scope="external">https://community.emc.com/docs/DOC-39522</xref>.
--->
- <!-- This is a little bit of a circular loop when this topic is conrefed into the main Isilon page,
- consider if there's a way to conditionalize it out in that case. -->
- For information about managing Isilon storage devices through Cloudera Manager, see
- <xref audience="integrated" href="cm_mc_isilon_service.xml"/><xref audience="standalone" href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_isilon_service.html" scope="external" format="html"/>.
- </p>
-
<!-- <p outputclass="toc inpage"/> -->
</conbody>
-<concept id="isilon_cm_configs" audience="hidden">
-<title>Required Configurations</title>
-<conbody>
-<p>Specify the following configurations in Cloudera Manager on the <menucascade><uicontrol>Clusters</uicontrol><uicontrol><varname>Isilon Service</varname></uicontrol><uicontrol>Configuration</uicontrol></menucascade> tab:<ul id="ul_vpx_bw5_vv">
-<li>In <uicontrol>HDFS Client Advanced Configuration Snippet (Safety Valve) for hdfs-site.xml</uicontrol> <codeph>hdfs-site.xml</codeph> and the <uicontrol>Cluster-wide Advanced Configuration Snippet (Safety Valve) for core-site.xml</uicontrol> properties for the Isilon service, set the value of the <codeph>dfs.client.file-block-storage-locations.timeout.millis</codeph> property to <codeph>10000</codeph>.</li>
-<li>In the Isilon <uicontrol>Cluster-wide Advanced Configuration Snippet (Safety Valve) for core-site.xml</uicontrol> property for the Isilon service, set the value of the <codeph>hadoop.security.token.service.use_ip</codeph> property to <codeph>FALSE</codeph>. </li>
-<li>If you see errors that reference the <codeph>.Trash</codeph> directory, make sure that the <uicontrol>Use Trash</uicontrol> property is selected.</li>
-</ul></p>
-
-</conbody>
-</concept>
</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_ldap.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_ldap.xml b/docs/topics/impala_ldap.xml
index 757a4e2..b01eb64 100644
--- a/docs/topics/impala_ldap.xml
+++ b/docs/topics/impala_ldap.xml
@@ -185,16 +185,6 @@ under the License.
</li>
</ul>
- <p audience="hidden">
- For clusters managed by Cloudera Manager 5.4.0 and higher,
- search for the configuration field names <codeph>ldap_domain</codeph>,
- <codeph>ldap_basedn</codeph>, or <codeph>ldap_bind_pattern</codeph>,
- fill in and save the appropriate field values, and restart the Impala service.
- Prior to Cloudera Manager 5.4.0, these values were filled in using the
- <uicontrol>Impala Daemon Command Line Argument Advanced Configuration Snippet (Safety Valve)</uicontrol>
- field.
- </p>
-
<p>
These options are mutually exclusive; Impala does not start if more than one of these options is specified.
</p>
@@ -299,26 +289,6 @@ username, for example <codeph>username@example.com</codeph>.
</metadata>
</prolog>
<conbody>
- <section id="ldap_impala_hue_cm" audience="hidden">
- <title>Enabling LDAP for Impala in Hue Using Cloudera Manager</title>
- <p>
- <ol>
- <li>Go to the Hue service.</li>
- <li>Click the Configuration tab.</li>
- <li>Select <menucascade><uicontrol>Scope</uicontrol><uicontrol>Hue
- Server</uicontrol></menucascade>.</li>
- <li>Select
- <menucascade><uicontrol>Category</uicontrol><uicontrol>Advanced</uicontrol></menucascade>.</li>
- <li>Add the following properties to the <b>Hue Server Advanced
- Configuration Snippet (Safety Valve) for
- hue_safety_valve_server.ini</b>
- property.<codeblock>[impala]
-auth_username=<LDAP username of Hue user to be authenticated>
-auth_password=<LDAP password of Hue user to be authenticated></codeblock></li>
- <li>Click <b>Save Changes</b>.</li>
- </ol>
- </p>
- </section>
<section id="ldap_impala_hue_cmdline">
<title>Enabling LDAP for Impala in Hue Using the Command Line</title>
<p>LDAP authentication for the Impala app in Hue can be enabled by
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_logging.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_logging.xml b/docs/topics/impala_logging.xml
index 92b9c88..79bc112 100644
--- a/docs/topics/impala_logging.xml
+++ b/docs/topics/impala_logging.xml
@@ -166,11 +166,6 @@ under the License.
important messages such as informational ones are buffered in memory rather than being flushed to disk
immediately.
</li>
-
- <li audience="hidden">
- Cloudera Manager has an Impala configuration setting that sets the <codeph>-logbuflevel</codeph> startup
- option.
- </li>
</ul>
</conbody>
@@ -189,15 +184,6 @@ under the License.
<conbody>
- <p rev="upstream" audience="hidden"><!-- Whole paragraph can probably go. -->
- <ph rev="upstream">Cloudera</ph> recommends installing Impala through the Cloudera Manager administration interface. To assist with
- troubleshooting, Cloudera Manager collects front-end and back-end logs together into a single view, and let
- you do a search across log data for all the managed nodes rather than examining the logs on each node
- separately. If you installed Impala using Cloudera Manager, refer to the topics on Monitoring Services
- (<xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_dg_service_monitoring.html" scope="external" format="html">CDH 5</xref>)
- or Logs (<xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_dg_logs.html" scope="external" format="html">CDH 5</xref>).
- </p>
-
<p>
If you are using Impala in an environment not managed by Cloudera Manager, review Impala log files on each
host, when you have traced an issue back to a specific system.
@@ -250,15 +236,6 @@ under the License.
Linux tool or technique of choice. A value of 1 preserves only the very latest log file.
</p>
- <p audience="hidden">
- To set up log rotation on a system managed by Cloudera Manager 5.4.0 and higher, search for the
- <codeph>max_log_files</codeph> option name and set the appropriate value for the <userinput>Maximum Log
- Files</userinput> field for each Impala configuration category (Impala, Catalog Server, and StateStore).
- Then restart the Impala service. In earlier Cloudera Manager releases, specify the
- <codeph>-max_log_files=<varname>maximum</varname></codeph> option in the <uicontrol>Command Line Argument
- Advanced Configuration Snippet (Safety Valve)</uicontrol> field for each Impala configuration category.
- </p>
-
</conbody>
</concept>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_perf_hdfs_caching.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_perf_hdfs_caching.xml b/docs/topics/impala_perf_hdfs_caching.xml
index 79d700a..9b16153 100644
--- a/docs/topics/impala_perf_hdfs_caching.xml
+++ b/docs/topics/impala_perf_hdfs_caching.xml
@@ -469,24 +469,6 @@ Found 122 entries
</li>
</ul>
- <p audience="hidden">
- <b>Cloudera Manager:</b>
- </p>
-
- <ul audience="hidden">
- <li>
- You can enable or disable HDFS caching through Cloudera Manager, using the configuration setting
- <uicontrol>Maximum Memory Used for Caching</uicontrol> for the HDFS service. This control sets the HDFS
- configuration parameter <codeph>dfs_datanode_max_locked_memory</codeph>, which specifies the upper limit
- of HDFS cache size on each node.
- </li>
-
- <li>
- All the other manipulation of the HDFS caching settings, such as what files are cached, is done through
- the command line, either Impala DDL statements or the Linux <cmdname>hdfs cacheadmin</cmdname> command.
- </li>
- </ul>
-
<p>
<b>Impala memory limits:</b>
</p>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_processes.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_processes.xml b/docs/topics/impala_processes.xml
index a97a644..71986d3 100644
--- a/docs/topics/impala_processes.xml
+++ b/docs/topics/impala_processes.xml
@@ -69,28 +69,6 @@ under the License.
<p outputclass="toc inpage"/>
</conbody>
- <concept id="starting_via_cm" audience="hidden">
-
- <title>Starting Impala through Cloudera Manager</title>
-
- <conbody>
-
- <p>
- If you installed Impala with Cloudera Manager, use Cloudera Manager to start and stop services. The
- Cloudera Manager GUI is a convenient way to check that all services are running, to set configuration
- options using form fields in a browser, and to spot potential issues such as low disk space before they
- become serious. Cloudera Manager automatically starts all the Impala-related services as a group, in the
- correct order. See
- <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_mc_start_stop_service.html" scope="external" format="html">the
- Cloudera Manager Documentation</xref> for details.
- </p>
-
- <note>
- <p conref="../shared/impala_common.xml#common/udf_persistence_restriction"/>
- </note>
- </conbody>
- </concept>
-
<concept id="starting_via_cmdline">
<title>Starting Impala from the Command Line</title>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_proxy.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_proxy.xml b/docs/topics/impala_proxy.xml
index d0474c5..bb8d029 100644
--- a/docs/topics/impala_proxy.xml
+++ b/docs/topics/impala_proxy.xml
@@ -143,15 +143,6 @@ under the License.
Run the load-balancing proxy server, pointing it at the configuration file that you set up.
</li>
- <li audience="hidden">
- On systems managed by Cloudera Manager, on the page
- <menucascade><uicontrol>Impala</uicontrol><uicontrol>Configuration</uicontrol><uicontrol>Impala Daemon
- Default Group</uicontrol></menucascade>, specify a value for the <uicontrol>Impala Daemons Load
- Balancer</uicontrol> field. Specify the address of the load balancer in
- <codeph><varname>host</varname>:<varname>port</varname></codeph> format. This setting lets Cloudera
- Manager route all appropriate Impala-related operations through the proxy server.
- </li>
-
<li>
For any scripts, jobs, or configuration settings for applications that formerly connected to a specific
datanode to run Impala SQL statements, change the connection information (such as the <codeph>-i</codeph>
@@ -269,13 +260,6 @@ under the License.
running the <cmdname>impalad</cmdname> daemon.
</li>
- <li rev="CDH-40363" audience="hidden">
- For a cluster managed by Cloudera Manager (5.4.2 or higher), fill in the Impala configuration setting
- <uicontrol>Impala Daemons Load Balancer</uicontrol> with the appropriate host:port combination.
- Then restart the Impala service.
- For systems using a recent level of Cloudera Manager, this is all the configuration you need; you can skip the remaining steps in this procedure.
- </li>
-
<li>
On systems not managed by Cloudera Manager, or systems using Cloudera Manager earlier than 5.4.2:
@@ -320,24 +304,6 @@ under the License.
configuration snippet, add: <codeblock>--principal=impala/<varname>proxy_host</varname>@<varname>realm</varname>
--be_principal=impala/<varname>actual_host</varname>@<varname>realm</varname>
--keytab_file=<varname>path_to_merged_keytab</varname></codeblock>
- <note audience="hidden">
- <p>On a cluster managed by Cloudera Manager 5.1 (or higher),
- when you set up Kerberos authentication using the wizard, you
- can choose to allow Cloudera Manager to deploy the
- <systemoutput>krb5.conf</systemoutput> on your cluster. In
- such a case, you do not need to explicitly modify safety valve
- parameters as directed above. </p>
- <p>Every host has a different <codeph>--be_principal</codeph>
- because the actual hostname is different on each host. </p>
- <p> Specify the fully qualified domain name (FQDN) for the proxy
- host, not the IP address. Use the exact FQDN as returned by a
- reverse DNS lookup for the associated IP address. </p>
- </note>
- </li>
-
- <li audience="hidden">
- On a cluster managed by Cloudera Manager, create a role group to set the configuration values from
- the preceding step on a per-host basis.
</li>
<li>
@@ -352,10 +318,6 @@ under the License.
Restart Impala to make the changes take effect. Follow the appropriate steps depending on whether you use
Cloudera Manager or not:
<ul>
- <li audience="hidden">
- On a cluster managed by Cloudera Manager, restart the Impala service.
- </li>
-
<li>
On a cluster not managed by Cloudera Manager, restart the <cmdname>impalad</cmdname> daemons on all
hosts in the cluster, as well as the <cmdname>statestored</cmdname> and <cmdname>catalogd</cmdname>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_scalability.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_scalability.xml b/docs/topics/impala_scalability.xml
index 9f7ce3c..2362110 100644
--- a/docs/topics/impala_scalability.xml
+++ b/docs/topics/impala_scalability.xml
@@ -169,7 +169,7 @@ Memory Usage: Additional Notes
<p audience="hidden">
Details to fill in in future: Impact of <q>load catalog in background</q> option.
- Changing timeouts. Related Cloudera Manager settings.
+ Changing timeouts.
</p>
<p>
@@ -469,11 +469,6 @@ Memory Usage: Additional Notes
query.
</li>
- <li audience="hidden">
- The <uicontrol>Impala Queries</uicontrol> dialog in Cloudera Manager. You can see the peak memory
- usage for a query, combined across all nodes in the cluster.
- </li>
-
<li>
The <uicontrol>Queries</uicontrol> tab in the Impala debug web user interface. Select the query to
examine and click the corresponding <uicontrol>Profile</uicontrol> link. This data breaks down the
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_ssl.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_ssl.xml b/docs/topics/impala_ssl.xml
index 4e0b722..726e040 100644
--- a/docs/topics/impala_ssl.xml
+++ b/docs/topics/impala_ssl.xml
@@ -42,156 +42,10 @@ under the License.
in the cluster. This feature is important when you also use other features such as Kerberos
authentication or Sentry authorization, where credentials are being
transmitted back and forth.
- <note type="important" id="CMCDH_EitherOK" audience="hidden">
- <ul id="ul_e2s_bcd_np">
- <li>You can use either Cloudera Manager or the following command-line
- instructions to complete this configuration.</li>
- <!-- Took out another too-specific conref, to the CDH minor version also in CDHVariables.xml. -->
- <li>This information applies specifically to the version of Impala
- shown in the HTML page header or on the PDF title page. If you use
- an earlier version of CDH, see the documentation for that version
- located at <xref
- href="http://www.cloudera.com/content/support/en/documentation.html"
- format="html" scope="external">Cloudera Documentation</xref>.</li>
- </ul>
- </note>
</p>
</conbody>
- <concept id="concept_gnk_2tt_qp" audience="hidden">
-
- <title>Using Cloudera Manager</title>
-
- <prolog>
- <metadata>
- <data name="Category" value="Cloudera Manager"/>
- </metadata>
- </prolog>
-
- <conbody>
-
- <p>
- To configure Impala to listen for Beeswax and HiveServer2 requests on TLS/SSL-secured ports:
- <ol id="ol_rnf_ftt_qp">
- <li>
- Open the Cloudera Manager Admin Console and go to the <uicontrol>Impala</uicontrol> service.
- </li>
-
- <!-- Formerly: conref="../shared/cm_common_elements.xml#cm/config_edit" -->
- <li>Click the <uicontrol>Configuration</uicontrol> tab.</li>
-
- <li>
- Select <menucascade><uicontrol>Scope</uicontrol><uicontrol>Impala (Service-Wide)</uicontrol></menucascade>.
- </li>
-
- <li>
- Select <menucascade><uicontrol>Category</uicontrol><uicontrol>Security</uicontrol></menucascade>.
- </li>
-
- <li>
- Edit the following properties:
- <table frame="all"
- id="table_drf_ftt_qp">
- <title>Impala SSL Properties</title>
- <tgroup cols="2">
- <colspec colname="c1" colnum="1" colwidth="1*"/>
- <colspec colname="c2" colnum="2" colwidth="2.5*"/>
- <thead>
- <row>
- <entry>
- Property
- </entry>
- <entry>
- Description
- </entry>
- </row>
- </thead>
- <tbody>
- <row>
- <entry>
- <b>Enable TLS/SSL for Impala Client Services</b>
- </entry>
- <entry>
- Encrypt communication between clients (like ODBC, JDBC, and the Impala shell) and the Impala daemon using Transport
- Layer Security (TLS) (formerly known as Secure Socket Layer (SSL)).
- </entry>
- </row>
- <row>
- <entry>
- <b>SSL/TLS Certificate for Clients</b>
- </entry>
- <entry>
- Local path to the X509 certificate that identifies the Impala daemon to clients during TLS/SSL connections. This
- file must be in PEM format.
- </entry>
- </row>
- <row>
- <entry>
- <b>SSL/TLS Private Key for Clients</b>
- </entry>
- <entry>
- Local path to the private key that matches the certificate specified in the Certificate for Clients. This file must be
- in PEM format.
- </entry>
- </row>
- <row>
- <entry>
- <b>SSL/TLS Private Key Password for Clients</b>
- </entry>
- <entry>
- A shell command for Impala to run on startup to retrieve the password for a password-protected private key file.
- The output of the command is truncated to a maximum of 1024 bytes, and any trailing whitespace (such as spaces
- or newline characters) is trimmed. If the command exits with an error, Impala does not start. If the password
- is incorrect, clients cannot connect to the server regardless of whether the public key is correct.
- </entry>
- </row>
- <row>
- <entry>
- <b>SSL/TLS CA Certificate</b>
- </entry>
- <entry>
- Must be specified for TLS/SSL encryption to be enabled for communication
- between internal Impala components.
- </entry>
- </row>
- <row>
- <entry>
- <b>SSL/TLS Certificate for <varname>Impala component</varname> Webserver</b>
- </entry>
- <entry>
- There are three of these configuration settings, one each for <q>Impala Daemon</q>,
- <q>Catalog Server</q>, and <q>Statestore</q>.
- Each of these Impala components has its own internal web server that powers the
- associated web UI with diagnostic information.
- The configuration setting represents the local path to the X509 certificate that
- identifies the web server to clients during TLS/SSL connections. This
- file must be in PEM format.
- </entry>
- </row>
- </tbody>
- </tgroup>
- </table>
- </li>
-
- <!-- Formerly: conref="../shared/cm_common_elements.xml#cm/save_changes_short" -->
- <li>Click <uicontrol>Save Changes</uicontrol> to commit the changes.</li>
-
- <li>
- Restart the Impala service.
- </li>
- </ol>
- </p>
-
- <p>
- For information on configuring TLS/SSL communication with the <codeph>impala-shell</codeph> interpreter, see
- <xref href="#concept_q1p_j2d_rp/secref"/>.
- </p>
-
- </conbody>
-
- </concept>
-
<concept id="concept_q1p_j2d_rp">
<title>Using the Command Line</title>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_txtfile.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_txtfile.xml b/docs/topics/impala_txtfile.xml
index 5d3f315..659d9f0 100644
--- a/docs/topics/impala_txtfile.xml
+++ b/docs/topics/impala_txtfile.xml
@@ -483,15 +483,6 @@ INSERT INTO csv SELECT * FROM other_file_format_table;</codeblock>
<ol>
<li>
<b>Prepare your systems to work with LZO by downloading and installing the appropriate libraries:</b>
- <p audience="hidden">
- <b>On systems managed by Cloudera Manager using parcels:</b>
- </p>
-
- <p audience="hidden">
- See the setup instructions for the LZO parcel in the Cloudera Manager documentation for
- <xref href="http://www.cloudera.com/documentation/enterprise/latest/topics/cm_ig_install_gpl_extras.html" scope="external" format="html">Cloudera
- Manager 5</xref>.
- </p>
<p>
<b>On systems managed by Cloudera Manager using packages, or not managed by Cloudera Manager:</b>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8b417c69/docs/topics/impala_webui.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_webui.xml b/docs/topics/impala_webui.xml
index 57d247c..176822a 100644
--- a/docs/topics/impala_webui.xml
+++ b/docs/topics/impala_webui.xml
@@ -104,11 +104,6 @@ under the License.
suspect is having problems.
</p>
- <note audience="hidden">
- To get a convenient picture of the health of all Impala nodes in a cluster, use the Cloudera Manager
- interface, which collects the low-level operational information from all Impala nodes, and presents a
- unified view of the entire cluster.
- </note>
</conbody>
<concept audience="hidden" id="webui_impalad_disabling">
[4/5] incubator-impala git commit: IMPALA-4966: Add flatbuffers to
build
Posted by jr...@apache.org.
IMPALA-4966: Add flatbuffers to build
FlatBuffers version 1.6.0 is already included in the toolchain. This
commit adds it to the build system.
Change-Id: I2ca255ddf08ac846b454bfa1470ed67b1338d2b0
Reviewed-on: http://gerrit.cloudera.org:8080/6180
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/60c1c6e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/60c1c6e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/60c1c6e8
Branch: refs/heads/master
Commit: 60c1c6e81b8705f493c6f9844e4c9a0aeb813721
Parents: 71eb569
Author: Dimitris Tsirogiannis <dt...@cloudera.com>
Authored: Tue Feb 28 13:33:11 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Mar 2 09:43:03 2017 +0000
----------------------------------------------------------------------
CMakeLists.txt | 9 +++++++
bin/bootstrap_toolchain.py | 8 +++---
bin/impala-config.sh | 1 +
cmake_modules/FindFlatBuffers.cmake | 43 ++++++++++++++++++++++++++++++++
4 files changed, 57 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/60c1c6e8/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 207fac4..cc21604 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -73,6 +73,7 @@ set_dep_root(BOOST)
set_dep_root(BREAKPAD)
set_dep_root(BZIP2)
set_dep_root(CRCUTIL)
+set_dep_root(FLATBUFFERS)
set_dep_root(GCC)
set_dep_root(GFLAGS)
set_dep_root(GLOG)
@@ -244,6 +245,14 @@ message(STATUS "Thrift library path: ${THRIFT_LIBS}")
message(STATUS "Thrift static library: ${THRIFT_STATIC_LIB}")
message(STATUS "Thrift compiler: ${THRIFT_COMPILER}")
+# find flatbuffers headers, lib and compiler
+find_package(FlatBuffers REQUIRED)
+include_directories(${FLATBUFFERS_INCLUDE_DIR})
+set(LIBS ${LIBS} ${FLATBUFFERS_LIBS})
+message(STATUS "FlatBuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}")
+message(STATUS "FlatBuffers library path: ${FLATBUFFERS_LIBS}")
+message(STATUS "FlatBuffers compiler: ${FLATBUFFERS_COMPILER}")
+
# find Snappy headers and libs
find_package(Snappy REQUIRED)
include_directories(${SNAPPY_INCLUDE_DIR})
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/60c1c6e8/bin/bootstrap_toolchain.py
----------------------------------------------------------------------
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index 349cd60..72d4041 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -349,10 +349,10 @@ if __name__ == "__main__":
if not os.path.exists(toolchain_root):
os.makedirs(toolchain_root)
- packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "crcutil", "gcc",
- "gflags", "glog", "gperftools", "gtest", "kudu", "libev", "llvm",
- ("llvm", "3.8.0-asserts-p1"), "lz4", "openldap", "protobuf", "rapidjson", "re2",
- "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"]
+ packages = ["avro", "binutils", "boost", "breakpad", "bzip2", "cmake", "crcutil",
+ "flatbuffers", "gcc", "gflags", "glog", "gperftools", "gtest", "kudu", "libev",
+ "llvm", ("llvm", "3.8.0-asserts-p1"), "lz4", "openldap", "protobuf", "rapidjson",
+ "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib"]
bootstrap(toolchain_root, packages)
# Download the CDH components if necessary.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/60c1c6e8/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 3b16013..69a368a 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -84,6 +84,7 @@ export IMPALA_BZIP2_VERSION=1.0.6-p2
export IMPALA_CMAKE_VERSION=3.2.3-p1
export IMPALA_CRCUTIL_VERSION=440ba7babeff77ffad992df3a10c767f184e946e
export IMPALA_CYRUS_SASL_VERSION=2.1.23
+export IMPALA_FLATBUFFERS_VERSION=1.6.0
export IMPALA_GCC_VERSION=4.9.2
export IMPALA_GFLAGS_VERSION=2.0
export IMPALA_GLOG_VERSION=0.3.2-p2
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/60c1c6e8/cmake_modules/FindFlatBuffers.cmake
----------------------------------------------------------------------
diff --git a/cmake_modules/FindFlatBuffers.cmake b/cmake_modules/FindFlatBuffers.cmake
new file mode 100644
index 0000000..d61a640
--- /dev/null
+++ b/cmake_modules/FindFlatBuffers.cmake
@@ -0,0 +1,43 @@
+##############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+##############################################################################
+
+# Find FLATBUFFERS (flatbuffers/include, libflatbuffers.a, flatc)
+# This module defines:
+# FLATBUFFERS_INCLUDE_DIR, directory containing headers
+# FLATBUFFERS_LIBS, path to flatbuffers's static library
+# FLATBUFFERS_COMPILER, path to flatc compiler
+
+find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/flatbuffers.h
+ PATHS ${FLATBUFFERS_ROOT}/include
+ NO_CMAKE_SYSTEM_PATH
+ NO_SYSTEM_ENVIRONMENT_PATH)
+
+find_library(FLATBUFFERS_LIBS libflatbuffers.a
+ PATHS ${FLATBUFFERS_ROOT}/lib
+ NO_CMAKE_SYSTEM_PATH
+ NO_SYSTEM_ENVIRONMENT_PATH)
+
+find_program(FLATBUFFERS_COMPILER flatc
+ PATHS ${FLATBUFFERS_ROOT}/bin
+ NO_CMAKE_SYSTEM_PATH
+ NO_SYSTEM_ENVIRONMENT_PATH)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(FLATBUFFERS REQUIRED_VARS
+ FLATBUFFERS_INCLUDE_DIR FLATBUFERS_LIBS FLATBUFFERS_COMPILER)
[2/5] incubator-impala git commit: Remove deadcode from frontend -
Part 1
Posted by jr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java b/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
index 667da34..65902ac 100644
--- a/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
+++ b/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
@@ -82,10 +82,6 @@ public class AvroSchemaConverter {
return converter.convertFieldSchemasImpl(fieldSchemas, schemaName);
}
- public static Schema convertTable(Table impalaTable) {
- return convertColumns(impalaTable.getColumns(), impalaTable.getFullName());
- }
-
private Schema convertColumnsImpl(List<Column> columns, String schemaName) {
List<Schema.Field> avroFields = Lists.newArrayList();
for (Column column: columns) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/HdfsCachingUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/HdfsCachingUtil.java b/fe/src/main/java/org/apache/impala/util/HdfsCachingUtil.java
index a52ad91..1846834 100644
--- a/fe/src/main/java/org/apache/impala/util/HdfsCachingUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/HdfsCachingUtil.java
@@ -408,15 +408,6 @@ public class HdfsCachingUtil {
}
/**
- * Check if the poolName matches the pool of the cache directive
- * identified by directiveId
- */
- public static boolean isSamePool(String poolName, Long directiveId)
- throws ImpalaRuntimeException {
- return poolName.equals(getCachePool(directiveId));
- }
-
- /**
* Helper method for frequent lookup of replication factor in the thrift caching
* structure.
*/
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/KuduUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/KuduUtil.java b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
index 75b11b6..073cf70 100644
--- a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
@@ -308,18 +308,6 @@ public class KuduUtil {
Type.fromThrift(boundaryVal.getType()).toSql()));
}
- /**
- * Parses a string of the form "a, b, c" and returns a set of values split by ',' and
- * stripped of the whitespace.
- */
- public static HashSet<String> parseKeyColumns(String cols) {
- return Sets.newHashSet(Splitter.on(",").trimResults().split(cols.toLowerCase()));
- }
-
- public static List<String> parseKeyColumnsAsList(String cols) {
- return Lists.newArrayList(Splitter.on(",").trimResults().split(cols.toLowerCase()));
- }
-
public static boolean isSupportedKeyType(org.apache.impala.catalog.Type type) {
return type.isIntegerType() || type.isStringType();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/NativeLogger.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/NativeLogger.java b/fe/src/main/java/org/apache/impala/util/NativeLogger.java
index ebba884..fc5b169 100644
--- a/fe/src/main/java/org/apache/impala/util/NativeLogger.java
+++ b/fe/src/main/java/org/apache/impala/util/NativeLogger.java
@@ -25,7 +25,6 @@ import org.slf4j.LoggerFactory;
* log4j log messages to be logged by glog.
*/
public class NativeLogger {
- private final static Logger LOG = LoggerFactory.getLogger(NativeLogger.class);
private static boolean loaded_ = false;
// Writes a log message to glog
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/RequestPoolService.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/RequestPoolService.java b/fe/src/main/java/org/apache/impala/util/RequestPoolService.java
index 1443e68..9976623 100644
--- a/fe/src/main/java/org/apache/impala/util/RequestPoolService.java
+++ b/fe/src/main/java/org/apache/impala/util/RequestPoolService.java
@@ -107,9 +107,6 @@ public class RequestPoolService {
// llama-site.xml but is Impala-specific and Llama does not use this.
final static String QUEUE_TIMEOUT_KEY = "impala.admission-control.pool-queue-timeout-ms";
- // Default value of the pool queue timeout (ms).
- final static int QUEUE_TIMEOUT_MS_DEFAULT = 60 * 1000;
-
// Key for the pool default query options. Query options are specified as a
// comma delimited string of 'key=value' pairs, e.g. 'key1=val1,key2=val2'.
// This is specified in the llama-site.xml but is Impala-specific and Llama does not
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
index a1290cc..65a443c 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
@@ -297,14 +297,6 @@ public class SentryPolicyService {
}
/**
- * Revokes a privilege from an existing role.
- */
- public void revokeRolePrivilege(User requestingUser, String roleName,
- TPrivilege privilege) throws ImpalaException {
- revokeRolePrivileges(requestingUser, roleName, Lists.newArrayList(privilege));
- }
-
- /**
* Revokes privileges from an existing role.
*
* @param requestingUser - The requesting user.
@@ -390,26 +382,6 @@ public class SentryPolicyService {
}
/**
- * Lists all roles granted to all groups a user belongs to.
- */
- public List<TSentryRole> listUserRoles(User requestingUser)
- throws ImpalaException {
- SentryServiceClient client = new SentryServiceClient();
- try {
- return Lists.newArrayList(client.get().listUserRoles(
- requestingUser.getShortName()));
- } catch (SentryAccessDeniedException e) {
- throw new AuthorizationException(String.format(ACCESS_DENIED_ERROR_MSG,
- requestingUser.getName(), "LIST_USER_ROLES"));
- } catch (SentryUserException e) {
- throw new InternalException(
- "Error making 'listUserRoles' RPC to Sentry Service: ", e);
- } finally {
- client.close();
- }
- }
-
- /**
* Lists all roles.
*/
public List<TSentryRole> listAllRoles(User requestingUser) throws ImpalaException {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
index 25ec45c..dfc2580 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationTest.java
@@ -80,10 +80,6 @@ import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
public class AuthorizationTest {
-
- private final static Logger LOG =
- LoggerFactory.getLogger(AuthorizationTest.class);
-
// Policy file has defined current user and 'test_user' have:
// ALL permission on 'tpch' database and 'newdb' database
// ALL permission on 'functional_seq_snap' database
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/test/java/org/apache/impala/planner/S3PlannerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/planner/S3PlannerTest.java b/fe/src/test/java/org/apache/impala/planner/S3PlannerTest.java
index f571ebe..49f0188 100644
--- a/fe/src/test/java/org/apache/impala/planner/S3PlannerTest.java
+++ b/fe/src/test/java/org/apache/impala/planner/S3PlannerTest.java
@@ -37,9 +37,6 @@ public class S3PlannerTest extends PlannerTestBase {
// The path that will replace the value of TEST_FS_S3A_NAME in file paths.
private static final Path S3A_CANONICAL_BUCKET = new Path("s3a://bucket");
- // The Hadoop configuration.
- private final Configuration CONF = new Configuration();
-
// The value of the FILESYSTEM_PREFIX environment variable.
private Path fsName;
[3/5] incubator-impala git commit: Remove deadcode from frontend -
Part 1
Posted by jr...@apache.org.
Remove deadcode from frontend - Part 1
Change-Id: I3fa810cf4eaf9f9ff545fe80a11bc62619039224
Reviewed-on: http://gerrit.cloudera.org:8080/6172
Reviewed-by: Alex Behm <al...@cloudera.com>
Reviewed-by: Ravi Shetye <ra...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/71eb569f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/71eb569f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/71eb569f
Branch: refs/heads/master
Commit: 71eb569f395dc01fe7b17ec851f69f4c2625306f
Parents: 8b417c6
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Mon Feb 27 15:04:32 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Mar 2 02:02:17 2017 +0000
----------------------------------------------------------------------
.../apache/impala/analysis/AggregateInfo.java | 31 -------
.../apache/impala/analysis/AnalyticExpr.java | 2 -
.../apache/impala/analysis/AnalyticInfo.java | 13 ---
.../org/apache/impala/analysis/Analyzer.java | 89 --------------------
.../apache/impala/analysis/BinaryPredicate.java | 2 -
.../impala/analysis/CollectionStructType.java | 1 -
.../impala/analysis/CompoundPredicate.java | 17 ----
.../apache/impala/analysis/CreateDbStmt.java | 1 -
.../impala/analysis/CreateFunctionStmtBase.java | 1 -
.../analysis/CreateOrAlterViewStmtBase.java | 3 -
.../impala/analysis/CreateTableLikeStmt.java | 1 -
.../apache/impala/analysis/DescribeDbStmt.java | 1 -
.../apache/impala/analysis/DescriptorTable.java | 14 ---
.../impala/analysis/DropFunctionStmt.java | 1 -
.../apache/impala/analysis/ExistsPredicate.java | 2 -
.../java/org/apache/impala/analysis/Expr.java | 17 ----
.../apache/impala/analysis/InlineViewRef.java | 4 -
.../org/apache/impala/analysis/InsertStmt.java | 3 -
.../apache/impala/analysis/LimitElement.java | 5 --
.../org/apache/impala/analysis/ModifyStmt.java | 2 -
.../apache/impala/analysis/PartitionDef.java | 1 +
.../impala/analysis/PartitionListItem.java | 37 --------
.../apache/impala/analysis/PartitionSpec.java | 5 --
.../impala/analysis/PartitionSpecBase.java | 6 --
.../java/org/apache/impala/analysis/Path.java | 5 --
.../org/apache/impala/analysis/Predicate.java | 1 -
.../apache/impala/analysis/RangePartition.java | 3 -
.../impala/analysis/ResetMetadataStmt.java | 1 -
.../apache/impala/analysis/SelectListItem.java | 1 -
.../org/apache/impala/analysis/SlotRef.java | 2 -
.../org/apache/impala/analysis/Subquery.java | 2 -
.../org/apache/impala/analysis/TableRef.java | 25 ------
.../authorization/PrivilegeRequestBuilder.java | 8 --
.../impala/catalog/AggregateFunction.java | 6 --
.../java/org/apache/impala/catalog/Catalog.java | 2 -
.../impala/catalog/CatalogObjectCache.java | 1 -
.../impala/catalog/CatalogServiceCatalog.java | 19 -----
.../impala/catalog/ColumnNotFoundException.java | 2 -
.../org/apache/impala/catalog/ColumnStats.java | 6 --
.../main/java/org/apache/impala/catalog/Db.java | 1 -
.../org/apache/impala/catalog/Function.java | 19 -----
.../org/apache/impala/catalog/HBaseTable.java | 8 --
.../apache/impala/catalog/HdfsPartition.java | 13 +--
.../impala/catalog/HdfsStorageDescriptor.java | 5 --
.../org/apache/impala/catalog/HdfsTable.java | 7 --
.../org/apache/impala/catalog/KuduTable.java | 2 -
.../catalog/PartitionNotFoundException.java | 2 -
.../apache/impala/catalog/PrimitiveType.java | 1 -
.../apache/impala/catalog/RolePrivilege.java | 1 -
.../apache/impala/catalog/ScalarFunction.java | 16 ----
.../org/apache/impala/catalog/ScalarType.java | 8 --
.../java/org/apache/impala/catalog/Table.java | 6 --
.../impala/catalog/TableNotFoundException.java | 2 -
.../java/org/apache/impala/catalog/Type.java | 20 -----
.../java/org/apache/impala/catalog/View.java | 2 -
.../org/apache/impala/common/RuntimeEnv.java | 2 -
.../hive/executor/ImpalaTextWritable.java | 2 -
.../apache/impala/planner/DataPartition.java | 2 -
.../impala/planner/DataSourceScanNode.java | 9 --
.../org/apache/impala/planner/ExchangeNode.java | 2 -
.../apache/impala/planner/HBaseScanNode.java | 3 -
.../org/apache/impala/planner/HashJoinNode.java | 2 -
.../apache/impala/planner/JoinBuildSink.java | 2 -
.../impala/planner/NestedLoopJoinNode.java | 2 -
.../org/apache/impala/planner/PlanFragment.java | 2 -
.../org/apache/impala/planner/PlanNode.java | 3 -
.../apache/impala/planner/PlannerContext.java | 3 -
.../org/apache/impala/planner/ValueRange.java | 1 -
.../org/apache/impala/service/JniFrontend.java | 5 --
.../apache/impala/util/AvroSchemaConverter.java | 4 -
.../org/apache/impala/util/HdfsCachingUtil.java | 9 --
.../java/org/apache/impala/util/KuduUtil.java | 12 ---
.../org/apache/impala/util/NativeLogger.java | 1 -
.../apache/impala/util/RequestPoolService.java | 3 -
.../apache/impala/util/SentryPolicyService.java | 28 ------
.../impala/analysis/AuthorizationTest.java | 4 -
.../apache/impala/planner/S3PlannerTest.java | 3 -
77 files changed, 3 insertions(+), 557 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
index 5dbde32..3e12ee1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AggregateInfo.java
@@ -266,7 +266,6 @@ public class AggregateInfo extends AggregateInfoBase {
public AggregateInfo getSecondPhaseDistinctAggInfo() {
return secondPhaseDistinctAggInfo_;
}
- public AggPhase getAggPhase() { return aggPhase_; }
public boolean isMerge() { return aggPhase_.isMerge(); }
public boolean isDistinctAgg() { return secondPhaseDistinctAggInfo_ != null; }
public ExprSubstitutionMap getIntermediateSmap() { return intermediateTupleSmap_; }
@@ -298,22 +297,6 @@ public class AggregateInfo extends AggregateInfoBase {
}
/**
- * Append ids of all slots that are being referenced in the process
- * of performing the aggregate computation described by this AggregateInfo.
- */
- public void getRefdSlots(List<SlotId> ids) {
- Preconditions.checkState(outputTupleDesc_ != null);
- if (groupingExprs_ != null) {
- Expr.getIds(groupingExprs_, null, ids);
- }
- Expr.getIds(aggregateExprs_, null, ids);
- // The backend assumes that the entire aggTupleDesc is materialized
- for (int i = 0; i < outputTupleDesc_.getSlots().size(); ++i) {
- ids.add(outputTupleDesc_.getSlots().get(i).getId());
- }
- }
-
- /**
* Substitute all the expressions (grouping expr, aggregate expr) and update our
* substitution map according to the given substitution map:
* - smap typically maps from tuple t1 to tuple t2 (example: the smap of an
@@ -717,20 +700,6 @@ public class AggregateInfo extends AggregateInfoBase {
}
}
- /**
- * Returns DataPartition derived from grouping exprs.
- * Returns unpartitioned spec if no grouping.
- * TODO: this won't work when we start supporting range partitions,
- * because we could derive both hash and order-based partitions
- */
- public DataPartition getPartition() {
- if (groupingExprs_.isEmpty()) {
- return DataPartition.UNPARTITIONED;
- } else {
- return DataPartition.hashPartitioned(groupingExprs_);
- }
- }
-
@Override
public String debugString() {
StringBuilder out = new StringBuilder(super.debugString());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
index 864c10b..bf0fb1b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalyticExpr.java
@@ -62,8 +62,6 @@ import com.google.common.collect.Lists;
* and need to be substituted as such; example: COUNT(COUNT(..)) OVER (..)
*/
public class AnalyticExpr extends Expr {
- private final static Logger LOG = LoggerFactory.getLogger(AnalyticExpr.class);
-
private FunctionCallExpr fnCall_;
private final List<Expr> partitionExprs_;
// These elements are modified to point to the corresponding child exprs to keep them
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/AnalyticInfo.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AnalyticInfo.java b/fe/src/main/java/org/apache/impala/analysis/AnalyticInfo.java
index 5e59e0f..26ce3f8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AnalyticInfo.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AnalyticInfo.java
@@ -125,19 +125,6 @@ public class AnalyticInfo extends AggregateInfoBase {
return result;
}
- /**
- * Append ids of all slots that are being referenced in the process
- * of performing the analytic computation described by this AnalyticInfo.
- */
- public void getRefdSlots(List<SlotId> ids) {
- Preconditions.checkState(intermediateTupleDesc_ != null);
- Expr.getIds(analyticExprs_, null, ids);
- // The backend assumes that the entire intermediateTupleDesc is materialized
- for (SlotDescriptor slotDesc: intermediateTupleDesc_.getSlots()) {
- ids.add(slotDesc.getId());
- }
- }
-
@Override
public void materializeRequiredSlots(Analyzer analyzer, ExprSubstitutionMap smap) {
materializedSlots_.clear();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 2809851..ef008b7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -165,7 +165,6 @@ public class Analyzer {
isSubquery_ = true;
globalState_.containsSubquery = true;
}
- public boolean isSubquery() { return isSubquery_; }
public boolean setHasPlanHints() { return globalState_.hasPlanHints = true; }
public boolean hasPlanHints() { return globalState_.hasPlanHints; }
public void setIsWithClause() { isWithClause_ = true; }
@@ -316,12 +315,6 @@ public class Analyzer {
public boolean containsSubquery() { return globalState_.containsSubquery; }
- /**
- * Helper function to reset the global state information about the existence of
- * subqueries.
- */
- public void resetSubquery() { globalState_.containsSubquery = false; }
-
// An analyzer stores analysis state for a single select block. A select block can be
// a top level select statement, or an inline view select block.
// ancestors contains the Analyzers of the enclosing select blocks of 'this'
@@ -1588,35 +1581,6 @@ public class Analyzer {
}
/**
- * Modifies the analysis state associated with the rhs table ref of an outer join
- * to accomodate a join inversion that changes the rhs table ref of the join from
- * oldRhsTbl to newRhsTbl.
- * TODO: Revisit this function and how outer joins are inverted. This function
- * should not be necessary because the semantics of an inverted outer join do
- * not change. This function will naturally become obsolete when we can transform
- * outer joins with otherPredicates into inner joins.
- */
- public void invertOuterJoinState(TableRef oldRhsTbl, TableRef newRhsTbl) {
- Preconditions.checkState(oldRhsTbl.getJoinOp().isOuterJoin());
- // Invert analysis state for an outer join.
- List<ExprId> conjunctIds =
- globalState_.conjunctsByOjClause.remove(oldRhsTbl.getId());
- if (conjunctIds != null) {
- globalState_.conjunctsByOjClause.put(newRhsTbl.getId(), conjunctIds);
- for (ExprId eid: conjunctIds) {
- globalState_.ojClauseByConjunct.put(eid, newRhsTbl);
- }
- } else {
- // An outer join is allowed not to have an On-clause if the rhs table ref is
- // correlated or relative.
- Preconditions.checkState(oldRhsTbl.isCorrelated() || oldRhsTbl.isRelative());
- }
- for (Map.Entry<TupleId, TableRef> e: globalState_.outerJoinedTupleIds.entrySet()) {
- if (e.getValue() == oldRhsTbl) e.setValue(newRhsTbl);
- }
- }
-
- /**
* For each equivalence class, adds/removes predicates from conjuncts such that it
* contains a minimum set of <lhsSlot> = <rhsSlot> predicates that establish the known
* equivalences between slots in lhsTids and rhsTids which must be disjoint.
@@ -2086,11 +2050,6 @@ public class Analyzer {
return globalState_.equivClassBySlotId.get(slotId);
}
- public Collection<SlotId> getEquivSlots(SlotId slotId) {
- EquivalenceClassId classId = globalState_.equivClassBySlotId.get(slotId);
- return globalState_.equivClassMembers.get(classId);
- }
-
public ExprSubstitutionMap getEquivClassSmap() { return globalState_.equivClassSmap; }
/**
@@ -2118,27 +2077,6 @@ public class Analyzer {
}
/**
- * Removes redundant expressions from exprs based on equivalence classes, as follows:
- * First, normalizes the exprs using the canonical SlotRef representative of each
- * equivalence class. Then retains the first original element of exprs that is
- * non-redundant in the normalized exprs. Returns a new list with the unique exprs.
- */
- public List<Expr> removeRedundantExprs(List<Expr> exprs) {
- List<Expr> result = Lists.newArrayList();
- List<Expr> normalizedExprs =
- Expr.substituteList(exprs, globalState_.equivClassSmap, this, false);
- Preconditions.checkState(exprs.size() == normalizedExprs.size());
- List<Expr> uniqueExprs = Lists.newArrayList();
- for (int i = 0; i < normalizedExprs.size(); ++i) {
- if (!uniqueExprs.contains(normalizedExprs.get(i))) {
- uniqueExprs.add(normalizedExprs.get(i));
- result.add(exprs.get(i).clone());
- }
- }
- return result;
- }
-
- /**
* Mark predicates as assigned.
*/
public void markConjunctsAssigned(List<Expr> conjuncts) {
@@ -2155,10 +2093,6 @@ public class Analyzer {
globalState_.assignedConjuncts.add(conjunct.getId());
}
- public boolean isConjunctAssigned(Expr conjunct) {
- return globalState_.assignedConjuncts.contains(conjunct.getId());
- }
-
public Set<ExprId> getAssignedConjuncts() {
return Sets.newHashSet(globalState_.assignedConjuncts);
}
@@ -2168,19 +2102,6 @@ public class Analyzer {
}
/**
- * Return true if there's at least one unassigned non-auxiliary conjunct.
- */
- public boolean hasUnassignedConjuncts() {
- for (ExprId id: globalState_.conjuncts.keySet()) {
- if (globalState_.assignedConjuncts.contains(id)) continue;
- Expr e = globalState_.conjuncts.get(id);
- if (e.isAuxExpr()) continue;
- return true;
- }
- return false;
- }
-
- /**
* Mark all slots that are referenced in exprs as materialized.
*/
public void materializeSlots(List<Expr> exprs) {
@@ -2486,10 +2407,6 @@ public class Analyzer {
return tableName.isFullyQualified() ? tableName.getDb() : getDefaultDb();
}
- public String getTargetDbName(FunctionName fnName) {
- return fnName.isFullyQualified() ? fnName.getDb() : getDefaultDb();
- }
-
/**
* Returns the fully-qualified table name of tableName. If tableName
* is already fully qualified, returns tableName.
@@ -2519,12 +2436,6 @@ public class Analyzer {
public List<Expr> getConjuncts() {
return new ArrayList<Expr>(globalState_.conjuncts.values());
}
- public Expr getConjunct(ExprId exprId) {
- return globalState_.conjuncts.get(exprId);
- }
- public Map<TupleId, List<ExprId>> getEqJoinConjuncts() {
- return globalState_.eqJoinConjuncts;
- }
public int incrementCallDepth() { return ++callDepth_; }
public int decrementCallDepth() { return --callDepth_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
index 145dbce..cc4b030 100644
--- a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
@@ -44,8 +44,6 @@ import com.google.common.collect.Lists;
*
*/
public class BinaryPredicate extends Predicate {
- private final static Logger LOG = LoggerFactory.getLogger(BinaryPredicate.class);
-
// true if this BinaryPredicate is inferred from slot equivalences, false otherwise.
private boolean isInferred_ = false;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java b/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java
index d05a6b2..a982774 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CollectionStructType.java
@@ -75,5 +75,4 @@ public class CollectionStructType extends StructType {
public StructField getOptionalField() { return optionalField_; }
public boolean isMapStruct() { return isMapStruct_; }
- public boolean isArrayStruct() { return !isMapStruct_; }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java b/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java
index 2655eaa..6354004 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CompoundPredicate.java
@@ -159,23 +159,6 @@ public class CompoundPredicate extends Predicate {
}
/**
- * Retrieve the slots bound by BinaryPredicate, InPredicate and
- * CompoundPredicates in the subtree rooted at 'this'.
- */
- public ArrayList<SlotRef> getBoundSlots() {
- ArrayList<SlotRef> slots = Lists.newArrayList();
- for (int i = 0; i < getChildren().size(); ++i) {
- if (getChild(i) instanceof BinaryPredicate ||
- getChild(i) instanceof InPredicate) {
- slots.add(((Predicate)getChild(i)).getBoundSlot());
- } else if (getChild(i) instanceof CompoundPredicate) {
- slots.addAll(((CompoundPredicate)getChild(i)).getBoundSlots());
- }
- }
- return slots;
- }
-
- /**
* Negates a CompoundPredicate.
*/
@Override
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
index a9fbf20..9b2fd10 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateDbStmt.java
@@ -58,7 +58,6 @@ public class CreateDbStmt extends StatementBase {
public String getComment() { return comment_; }
public String getDb() { return dbName_; }
public boolean getIfNotExists() { return ifNotExists_; }
- public HdfsUri getLocation() { return location_; }
@Override
public String toSql() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java b/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
index b9c49e5..211b3de 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateFunctionStmtBase.java
@@ -83,7 +83,6 @@ public abstract class CreateFunctionStmtBase extends StatementBase {
optArgs_ = optArgs;
}
- public String getComment() { return optArgs_.get(OptArg.COMMENT); }
public boolean getIfNotExists() { return ifNotExists_; }
public boolean hasSignature() { return args_ != null; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
index 1fba940..701964a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateOrAlterViewStmtBase.java
@@ -200,10 +200,7 @@ public abstract class CreateOrAlterViewStmtBase extends StatementBase {
return owner_;
}
- public List<ColumnDef> getColumnDescs() {return columnDefs_; }
- public String getComment() { return comment_; }
public boolean getIfNotExists() { return ifNotExists_; }
- public String getOriginalViewDef() { return originalViewDef_; }
public String getInlineViewDef() { return inlineViewDef_; }
public String getTbl() { return tableName_.getTbl(); }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
index 6fde627..b0aad63 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
@@ -76,7 +76,6 @@ public class CreateTableLikeStmt extends StatementBase {
public String getSrcTbl() { return srcTableName_.getTbl(); }
public boolean isExternal() { return isExternal_; }
public boolean getIfNotExists() { return ifNotExists_; }
- public String getComment() { return comment_; }
public THdfsFileFormat getFileFormat() { return fileFormat_; }
public HdfsUri getLocation() { return location_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java b/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
index 879e4e1..c6ffc9e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescribeDbStmt.java
@@ -55,7 +55,6 @@ public class DescribeDbStmt extends StatementBase {
}
public String getDb() { return dbName_; }
- public TDescribeOutputStyle getOutputStyle() { return outputStyle_; }
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
index 7040be6..f24a429 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DescriptorTable.java
@@ -101,12 +101,9 @@ public class DescriptorTable {
public TupleDescriptor getTupleDesc(TupleId id) { return tupleDescs_.get(id); }
public SlotDescriptor getSlotDesc(SlotId id) { return slotDescs_.get(id); }
- public Collection<TupleDescriptor> getTupleDescs() { return tupleDescs_.values(); }
public Collection<SlotDescriptor> getSlotDescs() { return slotDescs_.values(); }
- public TupleId getMaxTupleId() { return tupleIdGenerator_.getMaxId(); }
public SlotId getMaxSlotId() { return slotIdGenerator_.getMaxId(); }
- public Table getTargetTable() { return targetTable_; }
public void setTargetTable(Table table) { targetTable_ = table; }
/**
@@ -140,17 +137,6 @@ public class DescriptorTable {
}
/**
- * Return all ids in slotIds that belong to tupleId.
- */
- public List<SlotId> getTupleSlotIds(List<SlotId> slotIds, TupleId tupleId) {
- List<SlotId> result = Lists.newArrayList();
- for (SlotId id: slotIds) {
- if (getSlotDesc(id).getParent().getId().equals(tupleId)) result.add(id);
- }
- return result;
- }
-
- /**
* Computes physical layout parameters of all descriptors.
* Call this only after the last descriptor was added.
* Test-only.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java
index 9d9dd80..7a4fa53 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DropFunctionStmt.java
@@ -51,7 +51,6 @@ public class DropFunctionStmt extends StatementBase {
ifExists_ = ifExists;
}
- public FunctionName getFunction() { return desc_.getFunctionName(); }
public boolean getIfExists() { return ifExists_; }
private boolean hasSignature() { return fnArgs_ != null; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java b/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java
index c8ecfc9..355f562 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ExistsPredicate.java
@@ -28,8 +28,6 @@ import com.google.common.base.Preconditions;
* Class representing a [NOT] EXISTS predicate.
*/
public class ExistsPredicate extends Predicate {
- private final static Logger LOG = LoggerFactory.getLogger(
- ExistsPredicate.class);
private boolean notExists_ = false;
public boolean isNotExists() { return notExists_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/Expr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Expr.java b/fe/src/main/java/org/apache/impala/analysis/Expr.java
index d9c46bf..8f01005 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Expr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Expr.java
@@ -49,8 +49,6 @@ import com.google.common.collect.Sets;
*
*/
abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneable {
- private final static Logger LOG = LoggerFactory.getLogger(Expr.class);
-
// Limits on the number of expr children and the depth of an expr tree. These maximum
// values guard against crashes due to stack overflows (IMPALA-432) and were
// experimentally determined to be safe.
@@ -252,7 +250,6 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
public boolean isOnClauseConjunct() { return isOnClauseConjunct_; }
public void setIsOnClauseConjunct(boolean b) { isOnClauseConjunct_ = b; }
public boolean isAuxExpr() { return isAuxExpr_; }
- public boolean isRegisteredPredicate() { return id_ != null; }
public void setIsAuxExpr() { isAuxExpr_ = true; }
public Function getFn() { return fn_; }
@@ -917,13 +914,6 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
}
/**
- * Returns true if expr is fully bound by slotId, otherwise false.
- */
- public boolean isBound(SlotId slotId) {
- return isBoundBySlotIds(Lists.newArrayList(slotId));
- }
-
- /**
* Returns true if expr is fully bound by slotIds, otherwise false.
*/
public boolean isBoundBySlotIds(List<SlotId> slotIds) {
@@ -933,13 +923,6 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return true;
}
- public static boolean isBound(List<? extends Expr> exprs, List<TupleId> tids) {
- for (Expr expr: exprs) {
- if (!expr.isBoundByTupleIds(tids)) return false;
- }
- return true;
- }
-
public static Expr getFirstBoundChild(Expr expr, List<TupleId> tids) {
for (Expr child: expr.getChildren()) {
if (child.isBoundByTupleIds(tids)) return child;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 377ee1f..811d284 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -295,10 +295,6 @@ public class InlineViewRef extends TableRef {
}
public QueryStmt getViewStmt() { return queryStmt_; }
- public void setRewrittenViewStmt(QueryStmt stmt) {
- Preconditions.checkState(getAnalyzer().containsSubquery());
- queryStmt_ = stmt;
- }
public List<String> getExplicitColLabels() { return explicitColLabels_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index 902d100..916f97f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -51,8 +51,6 @@ import com.google.common.collect.Sets;
* whose results are to be inserted.
*/
public class InsertStmt extends StatementBase {
- private final static Logger LOG = LoggerFactory.getLogger(InsertStmt.class);
-
// Target table name as seen by the parser
private final TableName originalTableName_;
@@ -838,7 +836,6 @@ public class InsertStmt extends StatementBase {
* Only valid after analysis
*/
public QueryStmt getQueryStmt() { return queryStmt_; }
- public void setQueryStmt(QueryStmt stmt) { queryStmt_ = stmt; }
public List<Expr> getPartitionKeyExprs() { return partitionKeyExprs_; }
public boolean hasShuffleHint() { return hasShuffleHint_; }
public boolean hasNoShuffleHint() { return hasNoShuffleHint_; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/LimitElement.java b/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
index a8ef9e2..d357b07 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LimitElement.java
@@ -75,11 +75,6 @@ class LimitElement {
return limit_;
}
- public boolean hasLimit() {
- Preconditions.checkState(isAnalyzed_);
- return limit_ != -1;
- }
-
/**
* Returns the integer offset, evaluated from the offset expression. Must call
* analyze() first. If no offsetExpr exists, then 0 (the default offset) is returned.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
index 8463b31..c538968 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ModifyStmt.java
@@ -60,8 +60,6 @@ import com.google.common.collect.Sets;
*/
public abstract class ModifyStmt extends StatementBase {
- private final static org.slf4j.Logger LOG = LoggerFactory.getLogger(ModifyStmt.class);
-
// List of explicitly mentioned assignment expressions in the UPDATE's SET clause
protected final List<Pair<SlotRef, Expr>> assignments_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/PartitionDef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionDef.java b/fe/src/main/java/org/apache/impala/analysis/PartitionDef.java
index 6a1175d..b5443cb 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionDef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionDef.java
@@ -53,6 +53,7 @@ public class PartitionDef implements ParseNode {
}
public HdfsUri getLocation() { return location_; }
+
public PartitionSpec getPartitionSpec() { return partitionSpec_; }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/PartitionListItem.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionListItem.java b/fe/src/main/java/org/apache/impala/analysis/PartitionListItem.java
deleted file mode 100644
index 4870d28..0000000
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionListItem.java
+++ /dev/null
@@ -1,37 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.analysis;
-
-/**
- * Representation of a single column:value element in the PARTITION (...) clause of an insert
- * statement.
- */
-public class PartitionListItem {
- // Name of partitioning column.
- private final String colName_;
- // Value of partitioning column. Set to null for dynamic inserts.
- private final LiteralExpr value_;
-
- public PartitionListItem(String colName, LiteralExpr value) {
- this.colName_ = colName;
- this.value_ = value;
- }
-
- public String getColName() { return colName_; }
- public LiteralExpr getValue() { return value_; }
-}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
index 2ea53c4..c9b289f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
@@ -49,11 +49,6 @@ public class PartitionSpec extends PartitionSpecBase {
public List<PartitionKeyValue> getPartitionSpecKeyValues() { return partitionSpec_; }
- public boolean partitionExists() {
- Preconditions.checkNotNull(partitionExists_);
- return partitionExists_;
- }
-
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
index 5ead2d8..1a7c1a1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
@@ -37,8 +37,6 @@ public abstract class PartitionSpecBase implements ParseNode {
// Set during analysis.
protected String nullPartitionKeyValue_;
- public String getTbl() { return tableName_.getTbl(); }
-
public void setTableName(TableName tableName) {this.tableName_ = tableName; }
public TableName getTableName() { return tableName_; }
@@ -61,10 +59,6 @@ public abstract class PartitionSpecBase implements ParseNode {
return partitionShouldExist_ != null && partitionShouldExist_;
}
- public boolean getPartitionShouldNotExist() {
- return partitionShouldExist_ != null && !partitionShouldExist_;
- }
-
// Set the privilege requirement for this partition spec. Must be set prior to
// analysis.
public void setPrivilegeRequirement(Privilege privilege) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/Path.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Path.java b/fe/src/main/java/org/apache/impala/analysis/Path.java
index 89f3409..32ba3d5 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Path.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Path.java
@@ -282,11 +282,6 @@ public class Path {
return matchedTypes_;
}
- public List<Integer> getMatchedPositions() {
- Preconditions.checkState(isResolved_);
- return matchedPositions_;
- }
-
public boolean hasNonDestCollection() {
Preconditions.checkState(isResolved_);
return firstCollectionPathIdx_ != -1 &&
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/Predicate.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Predicate.java b/fe/src/main/java/org/apache/impala/analysis/Predicate.java
index d1311b7..34e6722 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Predicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Predicate.java
@@ -38,7 +38,6 @@ public abstract class Predicate extends Expr {
isEqJoinConjunct_ = other.isEqJoinConjunct_;
}
- public boolean isEqJoinConjunct() { return isEqJoinConjunct_; }
public void setIsEqJoinConjunct(boolean v) { isEqJoinConjunct_ = v; }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
index e2640a8..b441a78 100644
--- a/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
+++ b/fe/src/main/java/org/apache/impala/analysis/RangePartition.java
@@ -221,7 +221,4 @@ public class RangePartition implements ParseNode {
|| tRangePartition.isSetUpper_bound_values());
return tRangePartition;
}
-
- public List<Expr> getLowerBound() { return ImmutableList.copyOf(lowerBound_); }
- public List<Expr> getUpperBound() { return ImmutableList.copyOf(upperBound_); }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index f1c95ab..6f6a3f1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -50,7 +50,6 @@ public class ResetMetadataStmt extends StatementBase {
}
public TableName getTableName() { return tableName_; }
- public boolean isRefresh() { return isRefresh_; }
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/SelectListItem.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SelectListItem.java b/fe/src/main/java/org/apache/impala/analysis/SelectListItem.java
index b47e59f..74398a8 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SelectListItem.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SelectListItem.java
@@ -55,7 +55,6 @@ class SelectListItem {
public void setExpr(Expr expr) { expr_ = expr; }
public boolean isStar() { return isStar_; }
public String getAlias() { return alias_; }
- public void setAlias(String alias) { alias_ = alias; }
public List<String> getRawPath() { return rawPath_; }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
index 5f77f3b..21bfd22 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SlotRef.java
@@ -37,8 +37,6 @@ import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
public class SlotRef extends Expr {
- private final static Logger LOG = LoggerFactory.getLogger(SlotRef.class);
-
private final List<String> rawPath_;
private final String label_; // printed in toSql()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/Subquery.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Subquery.java b/fe/src/main/java/org/apache/impala/analysis/Subquery.java
index ab3132c..cc2c075 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Subquery.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Subquery.java
@@ -38,8 +38,6 @@ import com.google.common.collect.Sets;
* its own Analyzer context.
*/
public class Subquery extends Expr {
- private final static Logger LOG = LoggerFactory.getLogger(Subquery.class);
-
// The QueryStmt of the subquery.
protected QueryStmt stmt_;
// A subquery has its own analysis context
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/analysis/TableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableRef.java b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
index 975b70b..a56b8cd 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
@@ -263,7 +263,6 @@ public class TableRef implements ParseNode {
public List<PlanHint> getJoinHints() { return joinHints_; }
public List<PlanHint> getTableHints() { return tableHints_; }
public Expr getOnClause() { return onClause_; }
- public List<String> getUsingClause() { return usingColNames_; }
public void setJoinOp(JoinOperator op) { this.joinOp_ = op; }
public void setOnClause(Expr e) { this.onClause_ = e; }
public void setUsingClause(List<String> colNames) { this.usingColNames_ = colNames; }
@@ -571,30 +570,6 @@ public class TableRef implements ParseNode {
@Override
protected TableRef clone() { return new TableRef(this); }
- /**
- * Deep copies the given list of table refs and returns the clones in a new list.
- * The linking structure in the original table refs is preserved in the clones,
- * i.e., if the table refs were originally linked, then the corresponding clones
- * are linked in the same way. Similarly, if the original table refs were not linked
- * then the clones are also not linked.
- * Assumes that the given table refs are self-contained with respect to linking, i.e.,
- * that no table ref links to another table ref not in the list.
- */
- public static List<TableRef> cloneTableRefList(List<TableRef> tblRefs) {
- List<TableRef> clonedTblRefs = Lists.newArrayListWithCapacity(tblRefs.size());
- TableRef leftTblRef = null;
- for (TableRef tblRef: tblRefs) {
- TableRef tblRefClone = tblRef.clone();
- clonedTblRefs.add(tblRefClone);
- if (tblRef.leftTblRef_ != null) {
- Preconditions.checkState(tblRefs.contains(tblRef.leftTblRef_));
- tblRefClone.leftTblRef_ = leftTblRef;
- }
- leftTblRef = tblRefClone;
- }
- return clonedTblRefs;
- }
-
public void reset() {
isAnalyzed_ = false;
resolvedPath_ = null;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java b/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
index f47c52f..efe043d 100644
--- a/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
+++ b/fe/src/main/java/org/apache/impala/authorization/PrivilegeRequestBuilder.java
@@ -70,14 +70,6 @@ public class PrivilegeRequestBuilder {
}
/**
- * Sets the authorizeable object to be a URI.
- */
- public PrivilegeRequestBuilder onURI(String uriName) {
- authorizeable_ = new AuthorizeableUri(uriName);
- return this;
- }
-
- /**
* Specifies that permissions on any table in the given database.
*/
public PrivilegeRequestBuilder onAnyTable(String dbName) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/AggregateFunction.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/AggregateFunction.java b/fe/src/main/java/org/apache/impala/catalog/AggregateFunction.java
index ba8f972..bd87f80 100644
--- a/fe/src/main/java/org/apache/impala/catalog/AggregateFunction.java
+++ b/fe/src/main/java/org/apache/impala/catalog/AggregateFunction.java
@@ -44,8 +44,6 @@ public class AggregateFunction extends Function {
private String removeFnSymbol_;
private String finalizeFnSymbol_;
- private static String BE_BUILTINS_CLASS = "AggregateFunctions";
-
// If true, this aggregate function should ignore distinct.
// e.g. min(distinct col) == min(col).
// TODO: currently it is not possible for user functions to specify this. We should
@@ -174,8 +172,6 @@ public class AggregateFunction extends Function {
public String getInitFnSymbol() { return initFnSymbol_; }
public String getSerializeFnSymbol() { return serializeFnSymbol_; }
public String getMergeFnSymbol() { return mergeFnSymbol_; }
- public String getGetValueFnSymbol() { return getValueFnSymbol_; }
- public String getRemoveFnSymbol() { return removeFnSymbol_; }
public String getFinalizeFnSymbol() { return finalizeFnSymbol_; }
public boolean ignoresDistinct() { return ignoresDistinct_; }
public boolean isAnalyticFn() { return isAnalyticFn_; }
@@ -191,8 +187,6 @@ public class AggregateFunction extends Function {
public void setInitFnSymbol(String fn) { initFnSymbol_ = fn; }
public void setSerializeFnSymbol(String fn) { serializeFnSymbol_ = fn; }
public void setMergeFnSymbol(String fn) { mergeFnSymbol_ = fn; }
- public void setGetValueFnSymbol(String fn) { getValueFnSymbol_ = fn; }
- public void setRemoveFnSymbol(String fn) { removeFnSymbol_ = fn; }
public void setFinalizeFnSymbol(String fn) { finalizeFnSymbol_ = fn; }
public void setIntermediateType(Type t) { intermediateType_ = t; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/Catalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Catalog.java b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
index 403b9c1..5a3fc61 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
@@ -55,8 +55,6 @@ import org.apache.log4j.Logger;
* Builtins are populated on startup in initBuiltins().
*/
public abstract class Catalog {
- private static final Logger LOG = Logger.getLogger(Catalog.class);
-
// Initial catalog version.
public final static long INITIAL_CATALOG_VERSION = 0L;
public static final String DEFAULT_DB = "default";
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/CatalogObjectCache.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogObjectCache.java b/fe/src/main/java/org/apache/impala/catalog/CatalogObjectCache.java
index 9132905..c578e41 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogObjectCache.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogObjectCache.java
@@ -32,7 +32,6 @@ import com.google.common.collect.Lists;
* entries only get applied if the new/updated object has a larger catalog version.
*/
public class CatalogObjectCache<T extends CatalogObject> implements Iterable<T> {
- private static final Logger LOG = Logger.getLogger(CatalogObjectCache.class);
private final boolean caseInsensitiveKeys_;
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 00caf51..2adba9e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -1017,25 +1017,6 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
- * Drops the partition specified in 'partitionSpec' from 'tbl'. Throws a
- * CatalogException if 'tbl' is not an HdfsTable. If the partition having the given
- * partition spec does not exist, null is returned. Otherwise, the modified table is
- * returned.
- */
- public Table dropPartition(Table tbl, List<TPartitionKeyValue> partitionSpec)
- throws CatalogException {
- Preconditions.checkNotNull(tbl);
- Preconditions.checkNotNull(partitionSpec);
- Preconditions.checkArgument(tbl.getLock().isHeldByCurrentThread());
- if (!(tbl instanceof HdfsTable)) {
- throw new CatalogException("Table " + tbl.getFullName() + " is not an Hdfs table");
- }
- HdfsTable hdfsTable = (HdfsTable) tbl;
- if (hdfsTable.dropPartition(partitionSpec) == null) return null;
- return hdfsTable;
- }
-
- /**
* Adds a partition to its HdfsTable and returns the modified table.
*/
public Table addPartition(HdfsPartition partition) throws CatalogException {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/ColumnNotFoundException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ColumnNotFoundException.java b/fe/src/main/java/org/apache/impala/catalog/ColumnNotFoundException.java
index 80582b6..a865c4e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ColumnNotFoundException.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ColumnNotFoundException.java
@@ -26,6 +26,4 @@ public class ColumnNotFoundException extends CatalogException {
private static final long serialVersionUID = -2203080667446640542L;
public ColumnNotFoundException(String s) { super(s); }
-
- public ColumnNotFoundException(String s, Exception cause) { super(s, cause); }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java b/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
index 0b161df..ec4c0f5 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ColumnStats.java
@@ -40,8 +40,6 @@ import com.google.common.collect.Sets;
* Statistics for a single column.
*/
public class ColumnStats {
- private final static Logger LOG = LoggerFactory.getLogger(ColumnStats.class);
-
// Set of the currently supported column stats column types.
private final static Set<PrimitiveType> SUPPORTED_COL_TYPES = Sets.newHashSet(
PrimitiveType.BIGINT, PrimitiveType.BINARY, PrimitiveType.BOOLEAN,
@@ -157,9 +155,6 @@ public class ColumnStats {
return this;
}
- public void setAvgSize(float avgSize) { avgSize_ = avgSize; }
- public void setAvgSerializedSize(float avgSize) { avgSerializedSize_ = avgSize; }
- public void setMaxSize(long maxSize) { maxSize_ = maxSize; }
public long getNumDistinctValues() { return numDistinctValues_; }
public void setNumDistinctValues(long numDistinctValues) {
this.numDistinctValues_ = numDistinctValues;
@@ -171,7 +166,6 @@ public class ColumnStats {
public boolean hasNulls() { return numNulls_ > 0; }
public long getNumNulls() { return numNulls_; }
public boolean hasAvgSerializedSize() { return avgSerializedSize_ >= 0; }
- public boolean hasMaxSize() { return maxSize_ >= 0; }
public boolean hasNumDistinctValues() { return numDistinctValues_ >= 0; }
public boolean hasStats() { return numNulls_ != -1 || numDistinctValues_ != -1; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/Db.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Db.java b/fe/src/main/java/org/apache/impala/catalog/Db.java
index 0ed67c6..d8f5719 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Db.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Db.java
@@ -493,7 +493,6 @@ public class Db implements CatalogObject {
public long getCatalogVersion() { return catalogVersion_; }
@Override
public void setCatalogVersion(long newVersion) { catalogVersion_ = newVersion; }
- public Catalog getParentCatalog() { return parentCatalog_; }
@Override
public boolean isLoaded() { return true; }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/Function.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Function.java b/fe/src/main/java/org/apache/impala/catalog/Function.java
index 74df8c9..f7f2632 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Function.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Function.java
@@ -163,7 +163,6 @@ public class Function implements CatalogObject {
return argTypes_[argTypes_.length - 1];
}
- public void setName(FunctionName name) { name_ = name; }
public void setLocation(HdfsUri loc) { location_ = loc; }
public void setBinaryType(TFunctionBinaryType type) { binaryType_ = type; }
public void setHasVarArgs(boolean v) { hasVarArgs_ = v; }
@@ -240,24 +239,6 @@ public class Function implements CatalogObject {
return new Function(name_, promoted, retType_, hasVarArgs_);
}
- /**
- * Given a list of functions which are a super type of this function, select the best
- * match. This is the one which requires the fewest type promotions.
- */
- public Function selectClosestSuperType(List<Function> candidates) {
- Preconditions.checkArgument(candidates.size() > 0);
- if (candidates.size() == 1) return candidates.get(0);
-
- // Always promote CHAR to STRING before attempting any other promotions.
- Function withStrs = promoteCharsToStrings();
- for (Function f: candidates) {
- if (withStrs.isIndistinguishable(f)) return f;
- }
- // Otherwise, we use the previous rules of resolution which are to take the first
- // one in the list.
- return candidates.get(0);
- }
-
private boolean isIdentical(Function o) {
if (!o.name_.equals(name_)) return false;
if (o.argTypes_.length != this.argTypes_.length) return false;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
index 3af1160..c901f1d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
@@ -99,10 +99,6 @@ public class HBaseTable extends Table {
// Minimum number of regions that are checked to estimate the row count
private static final int MIN_NUM_REGIONS_TO_CHECK = 5;
- // Column referring to HBase row key.
- // Hive (including metastore) currently doesn't support composite HBase keys.
- protected HBaseColumn rowKey_;
-
// Name of table in HBase.
// 'this.name' is the alias of the HBase table in Hive.
protected String hbaseTableName_;
@@ -674,10 +670,6 @@ public class HBaseTable extends Table {
return hbaseTableName_;
}
- public static Configuration getHBaseConf() {
- return hbaseConf_;
- }
-
public int getNumNodes() {
// TODO: implement
return 100;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
index 5db2247..5cde346 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
@@ -209,8 +209,6 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
public long getOffset() { return fileBlock_.getOffset(); }
public long getLength() { return fileBlock_.getLength(); }
- // Returns true if at there at least one cached replica.
- public boolean isCached() { return isCached_; }
public List<Integer> getReplicaHostIdxs() {
return fileBlock_.getReplica_host_idxs();
}
@@ -441,16 +439,9 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
*/
public TAccessLevel getAccessLevel() { return accessLevel_; }
- /**
- * Returns the HMS parameter with key 'key' if it exists, otherwise returns null.
- */
- public String getParameter(String key) {
- return hmsParameters_.get(key);
- }
-
- public Map<String, String> getParameters() { return hmsParameters_; }
+ public Map<String, String> getParameters() { return hmsParameters_; }
- public void putToParameters(String k, String v) { hmsParameters_.put(k, v); }
+ public void putToParameters(String k, String v) { hmsParameters_.put(k, v); }
/**
* Marks this partition's metadata as "dirty" indicating that changes have been
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
index ace00c2..4554a31 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
@@ -42,7 +42,6 @@ public class HdfsStorageDescriptor {
// Serde parameters that are recognized by table writers.
private static final String BLOCK_SIZE = "blocksize";
- private static final String COMPRESSION = "compression";
// Important: don't change the ordering of these keys - if e.g. FIELD_DELIM is not
// found, the value of LINE_DELIM is used, so LINE_DELIM must be found first.
@@ -183,9 +182,6 @@ public class HdfsStorageDescriptor {
public InvalidStorageDescriptorException(Exception ex) {
super(ex.getMessage(), ex);
}
- public InvalidStorageDescriptorException(String msg, Throwable cause) {
- super(msg, cause);
- }
}
/**
@@ -234,7 +230,6 @@ public class HdfsStorageDescriptor {
public byte getCollectionDelim() { return collectionDelim_; }
public byte getMapKeyDelim() { return mapKeyDelim_; }
public byte getEscapeChar() { return escapeChar_; }
- public byte getQuoteChar() { return quoteChar_; }
public HdfsFileFormat getFileFormat() { return fileFormat_; }
public int getBlockSize() { return blockSize_; }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index fa6a1b7..3e51cba 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -212,8 +212,6 @@ public class HdfsTable extends Table {
new HdfsPartitionLocationCompressor(numClusteringCols_);
}
- public boolean spansMultipleFileSystems() { return multipleFileSystems_; }
-
/**
* Returns true if the table resides at a location which supports caching (e.g. HDFS).
*/
@@ -487,9 +485,6 @@ public class HdfsTable extends Table {
public Collection<HdfsPartition> getPartitions() { return partitionMap_.values(); }
public Map<Long, HdfsPartition> getPartitionMap() { return partitionMap_; }
- public Map<String, HdfsPartition> getNameToPartitionMap() {
- return nameToPartitionMap_;
- }
public Set<Long> getNullPartitionIds(int i) { return nullPartitionIds_.get(i); }
public HdfsPartitionLocationCompressor getPartitionLocationCompressor() {
return partitionLocationCompressor_;
@@ -504,7 +499,6 @@ public class HdfsTable extends Table {
* Set during load.
*/
public String getNullPartitionKeyValue() { return nullPartitionKeyValue_; }
- public String getNullColumnValue() { return nullColumnValue_; }
/*
* Returns the storage location (HDFS path) of this table.
@@ -1672,7 +1666,6 @@ public class HdfsTable extends Table {
return hdfsTable;
}
- public long getNumHdfsFiles() { return numHdfsFiles_; }
public long getTotalHdfsBytes() { return totalHdfsBytes_; }
public String getHdfsBaseDir() { return hdfsBaseDir_; }
public Path getHdfsBaseDirPath() { return new Path(hdfsBaseDir_); }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 8409b45..c58c665 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -67,8 +67,6 @@ import com.google.common.collect.Lists;
*/
public class KuduTable extends Table {
- private static final Logger LOG = Logger.getLogger(KuduTable.class);
-
// Alias to the string key that identifies the storage handler for Kudu tables.
public static final String KEY_STORAGE_HANDLER =
hive_metastoreConstants.META_TABLE_STORAGE;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/PartitionNotFoundException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/PartitionNotFoundException.java b/fe/src/main/java/org/apache/impala/catalog/PartitionNotFoundException.java
index 60707ef..3cd4afa 100644
--- a/fe/src/main/java/org/apache/impala/catalog/PartitionNotFoundException.java
+++ b/fe/src/main/java/org/apache/impala/catalog/PartitionNotFoundException.java
@@ -26,6 +26,4 @@ public class PartitionNotFoundException extends CatalogException {
private static final long serialVersionUID = -2203080667446640542L;
public PartitionNotFoundException(String s) { super(s); }
-
- public PartitionNotFoundException(String s, Exception cause) { super(s, cause); }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/PrimitiveType.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/PrimitiveType.java b/fe/src/main/java/org/apache/impala/catalog/PrimitiveType.java
index 986c4fe..7397734 100644
--- a/fe/src/main/java/org/apache/impala/catalog/PrimitiveType.java
+++ b/fe/src/main/java/org/apache/impala/catalog/PrimitiveType.java
@@ -101,5 +101,4 @@ public enum PrimitiveType {
}
public int getSlotSize() { return slotSize_; }
- public static int getMaxSlotSize() { return DECIMAL.slotSize_; }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java b/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
index 58bb906..6ce035f 100644
--- a/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
+++ b/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
@@ -134,5 +134,4 @@ public class RolePrivilege implements CatalogObject {
public long getCreateTimeMs() {
return privilege_.isSetCreate_time_ms() ? privilege_.getCreate_time_ms() : -1L;
}
- public TPrivilegeScope getScope() { return privilege_.getScope(); }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/ScalarFunction.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ScalarFunction.java b/fe/src/main/java/org/apache/impala/catalog/ScalarFunction.java
index 766111e..dd88a71 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ScalarFunction.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ScalarFunction.java
@@ -236,20 +236,6 @@ public class ScalarFunction extends Function {
}
/**
- * Create a function that is used to search the catalog for a matching builtin. Only
- * the fields necessary for matching function prototypes are specified.
- */
- public static ScalarFunction createBuiltinSearchDesc(String name, Type[] argTypes,
- boolean hasVarArgs) {
- ArrayList<Type> fnArgs =
- (argTypes == null) ? new ArrayList<Type>() : Lists.newArrayList(argTypes);
- ScalarFunction fn = new ScalarFunction(
- new FunctionName(Catalog.BUILTINS_DB, name), fnArgs, Type.INVALID, hasVarArgs);
- fn.setBinaryType(TFunctionBinaryType.BUILTIN);
- return fn;
- }
-
- /**
* Static helper method to create a scalar function of given
* TFunctionBinaryType.
*/
@@ -269,8 +255,6 @@ public class ScalarFunction extends Function {
public void setCloseFnSymbol(String s) { closeFnSymbol_ = s; }
public String getSymbolName() { return symbolName_; }
- public String getPrepareFnSymbol() { return prepareFnSymbol_; }
- public String getCloseFnSymbol() { return closeFnSymbol_; }
@Override
public String toSql(boolean ifNotExists) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
index 297f90e..f43cf6c 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ScalarType.java
@@ -242,14 +242,6 @@ public class ScalarType extends Type {
}
}
- public static Type[] toColumnType(PrimitiveType[] types) {
- Type result[] = new Type[types.length];
- for (int i = 0; i < types.length; ++i) {
- result[i] = createType(types[i]);
- }
- return result;
- }
-
public int decimalPrecision() {
Preconditions.checkState(type_ == PrimitiveType.DECIMAL);
return precision_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/Table.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java b/fe/src/main/java/org/apache/impala/catalog/Table.java
index 61289a5..f70186e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Table.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Table.java
@@ -65,7 +65,6 @@ public abstract class Table implements CatalogObject {
protected final Db db_;
protected final String name_;
protected final String owner_;
- protected TTableDescriptor tableDesc_;
protected TAccessLevel accessLevel_ = TAccessLevel.READ_WRITE;
// Lock protecting this table
private final ReentrantLock tableLock_ = new ReentrantLock();
@@ -89,10 +88,6 @@ public abstract class Table implements CatalogObject {
// The lastDdlTime for this table; -1 if not set
protected long lastDdlTime_;
- // Set of supported table types.
- protected static EnumSet<TableType> SUPPORTED_TABLE_TYPES = EnumSet.of(
- TableType.EXTERNAL_TABLE, TableType.MANAGED_TABLE, TableType.VIRTUAL_VIEW);
-
protected Table(org.apache.hadoop.hive.metastore.api.Table msTable, Db db,
String name, String owner) {
msTable_ = msTable;
@@ -376,7 +371,6 @@ public abstract class Table implements CatalogObject {
return new TableName(db_ != null ? db_.getName() : null, name_);
}
- public String getOwner() { return owner_; }
public ArrayList<Column> getColumns() { return colsByPos_; }
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/TableNotFoundException.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/TableNotFoundException.java b/fe/src/main/java/org/apache/impala/catalog/TableNotFoundException.java
index 7b93a20..3095939 100644
--- a/fe/src/main/java/org/apache/impala/catalog/TableNotFoundException.java
+++ b/fe/src/main/java/org/apache/impala/catalog/TableNotFoundException.java
@@ -26,6 +26,4 @@ public class TableNotFoundException extends CatalogException {
private static final long serialVersionUID = -2203080667446640542L;
public TableNotFoundException(String s) { super(s); }
-
- public TableNotFoundException(String s, Exception cause) { super(s, cause); }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/Type.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Type.java b/fe/src/main/java/org/apache/impala/catalog/Type.java
index 45848c5..3061194 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Type.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Type.java
@@ -154,7 +154,6 @@ public abstract class Type {
public boolean isBoolean() { return isScalarType(PrimitiveType.BOOLEAN); }
public boolean isTimestamp() { return isScalarType(PrimitiveType.TIMESTAMP); }
public boolean isDecimal() { return isScalarType(PrimitiveType.DECIMAL); }
- public boolean isDecimalOrNull() { return isDecimal() || isNull(); }
public boolean isFullySpecifiedDecimal() { return false; }
public boolean isWildcardDecimal() { return false; }
public boolean isWildcardVarchar() { return false; }
@@ -316,15 +315,6 @@ public abstract class Type {
}
/**
- * Checks if types t1 and t2 are assignment compatible, i.e. if both t1 and t2 can be
- * assigned to a type t without an explicit cast and without any conversions that would
- * result in loss of precision.
- */
- public static boolean areAssignmentCompatibleTypes(Type t1, Type t2) {
- return getAssignmentCompatibleType(t1, t2, true) != ScalarType.INVALID;
- }
-
- /**
* Returns true if this type exceeds the MAX_NESTING_DEPTH, false otherwise.
*/
public boolean exceedsMaxNestingDepth() { return exceedsMaxNestingDepth(0); }
@@ -447,16 +437,6 @@ public abstract class Type {
}
/**
- * Utility function to get the primitive type of a thrift type that is known
- * to be scalar.
- */
- public TPrimitiveType getTPrimitiveType(TColumnType ttype) {
- Preconditions.checkState(ttype.getTypesSize() == 1);
- Preconditions.checkState(ttype.types.get(0).getType() == TTypeNodeType.SCALAR);
- return ttype.types.get(0).scalar_type.getType();
- }
-
- /**
* JDBC data type description
* Returns the column size for this type.
* For numeric data this is the maximum precision.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/catalog/View.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/View.java b/fe/src/main/java/org/apache/impala/catalog/View.java
index f437c58..b82c489 100644
--- a/fe/src/main/java/org/apache/impala/catalog/View.java
+++ b/fe/src/main/java/org/apache/impala/catalog/View.java
@@ -164,8 +164,6 @@ public class View extends Table {
@Override
public TCatalogObjectType getCatalogObjectType() { return TCatalogObjectType.VIEW; }
public QueryStmt getQueryStmt() { return queryStmt_; }
- public String getOriginalViewDef() { return originalViewDef_; }
- public String getInlineViewDef() { return inlineViewDef_; }
public boolean isLocalView() { return isLocalView_; }
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
index d33455a..bfb079e 100644
--- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
+++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
@@ -28,8 +28,6 @@ import org.apache.impala.service.FeSupport;
* can rely on a machine-independent RuntimeEnv.
*/
public class RuntimeEnv {
- private final static Logger LOG = LoggerFactory.getLogger(RuntimeEnv.class);
-
public static RuntimeEnv INSTANCE = new RuntimeEnv();
private int numCores_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/hive/executor/ImpalaTextWritable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/hive/executor/ImpalaTextWritable.java b/fe/src/main/java/org/apache/impala/hive/executor/ImpalaTextWritable.java
index bae4537..2a87454 100644
--- a/fe/src/main/java/org/apache/impala/hive/executor/ImpalaTextWritable.java
+++ b/fe/src/main/java/org/apache/impala/hive/executor/ImpalaTextWritable.java
@@ -37,8 +37,6 @@ public class ImpalaTextWritable extends Text {
@Override
public int getLength() { return string_.getLength(); }
- public ImpalaStringWritable getStringWritable() { return string_; }
-
@Override
public void set(byte[] v, int offset, int len) {
string_.set(v, offset, len);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/DataPartition.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/DataPartition.java b/fe/src/main/java/org/apache/impala/planner/DataPartition.java
index f0f16fa..9e942e4 100644
--- a/fe/src/main/java/org/apache/impala/planner/DataPartition.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataPartition.java
@@ -40,8 +40,6 @@ import com.google.common.collect.Lists;
* DataStreamSender/DataStreamMgr/DataStreamRecvr).
*/
public class DataPartition {
- private final static Logger LOG = LoggerFactory.getLogger(DataPartition.class);
-
private final TPartitionType type_;
// for hash partition: exprs used to compute hash value
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
index 520f0f3..89f8377 100644
--- a/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/DataSourceScanNode.java
@@ -336,15 +336,6 @@ public class DataSourceScanNode extends ScanNode {
perHostMemCost_ = 1024L * 1024L * 1024L;
}
- /**
- * Returns the per-host upper bound of memory that any number of concurrent scan nodes
- * will use. Used for estimating the per-host memory requirement of queries.
- */
- public static long getPerHostMemUpperBound() {
- // TODO: What's a good estimate of memory consumption?
- return 1024L * 1024L * 1024L;
- }
-
@Override
protected String getNodeExplainString(String prefix, String detailPrefix,
TExplainLevel detailLevel) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/ExchangeNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/ExchangeNode.java b/fe/src/main/java/org/apache/impala/planner/ExchangeNode.java
index 658aa76..32673ae 100644
--- a/fe/src/main/java/org/apache/impala/planner/ExchangeNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/ExchangeNode.java
@@ -45,8 +45,6 @@ import com.google.common.base.Preconditions;
* inputs are also sorted individually on the same SortInfo parameter.
*/
public class ExchangeNode extends PlanNode {
- private final static Logger LOG = LoggerFactory.getLogger(ExchangeNode.class);
-
// The serialization overhead per tuple in bytes when sent over an exchange.
// Currently it accounts only for the tuple_offset entry per tuple (4B) in a
// BE TRowBatch. If we modify the RowBatch serialization, then we need to
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java b/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
index 9e25655..aea3750 100644
--- a/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HBaseScanNode.java
@@ -100,9 +100,6 @@ public class HBaseScanNode extends ScanNode {
private final static int DEFAULT_SUGGESTED_CACHING = 1024;
private int suggestedCaching_ = DEFAULT_SUGGESTED_CACHING;
- // HBase config; Common across all object instance.
- private static Configuration hbaseConf_ = HBaseConfiguration.create();
-
public HBaseScanNode(PlanNodeId id, TupleDescriptor desc) {
super(id, desc, "SCAN HBASE");
desc_ = desc;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java b/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java
index 07cfe9f..008cd5c 100644
--- a/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/HashJoinNode.java
@@ -47,8 +47,6 @@ import com.google.common.collect.Lists;
* inversion (for outer/semi/cross joins) it could also be the left child.
*/
public class HashJoinNode extends JoinNode {
- private final static Logger LOG = LoggerFactory.getLogger(HashJoinNode.class);
-
public HashJoinNode(PlanNode outer, PlanNode inner, boolean isStraightJoin,
DistributionMode distrMode, JoinOperator joinOp,
List<BinaryPredicate> eqJoinConjuncts, List<Expr> otherJoinConjuncts) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/JoinBuildSink.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/JoinBuildSink.java b/fe/src/main/java/org/apache/impala/planner/JoinBuildSink.java
index e214fa6..7afced6 100644
--- a/fe/src/main/java/org/apache/impala/planner/JoinBuildSink.java
+++ b/fe/src/main/java/org/apache/impala/planner/JoinBuildSink.java
@@ -41,8 +41,6 @@ import com.google.common.collect.Lists;
* Sink to materialize the build side of a join.
*/
public class JoinBuildSink extends DataSink {
- private final static Logger LOG = LoggerFactory.getLogger(JoinBuildSink.class);
-
// id of join's build-side table assigned during planning
private final JoinTableId joinTableId_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/NestedLoopJoinNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/NestedLoopJoinNode.java b/fe/src/main/java/org/apache/impala/planner/NestedLoopJoinNode.java
index cbaf370..0213c8b 100644
--- a/fe/src/main/java/org/apache/impala/planner/NestedLoopJoinNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/NestedLoopJoinNode.java
@@ -50,8 +50,6 @@ import com.google.common.base.Preconditions;
* right input.
*/
public class NestedLoopJoinNode extends JoinNode {
- private final static Logger LOG = LoggerFactory.getLogger(NestedLoopJoinNode.class);
-
public NestedLoopJoinNode(PlanNode outer, PlanNode inner, boolean isStraightJoin,
DistributionMode distrMode, JoinOperator joinOp, List<Expr> otherJoinConjuncts) {
super(outer, inner, isStraightJoin, distrMode, joinOp,
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
index 168e485..e4888f9 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanFragment.java
@@ -74,8 +74,6 @@ import com.google.common.collect.Sets;
* fix that
*/
public class PlanFragment extends TreeNode<PlanFragment> {
- private final static Logger LOG = LoggerFactory.getLogger(PlanFragment.class);
-
private final PlanFragmentId fragmentId_;
private PlanId planId_;
private CohortId cohortId_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/PlanNode.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlanNode.java b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
index 312ea5b..978e0ac 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlanNode.java
@@ -65,9 +65,6 @@ import com.google.common.math.LongMath;
abstract public class PlanNode extends TreeNode<PlanNode> {
private final static Logger LOG = LoggerFactory.getLogger(PlanNode.class);
- // TODO: Retrieve from the query options instead of using a default.
- protected final static int DEFAULT_BATCH_SIZE = 1024;
-
// String used for this node in getExplainString().
protected String displayName_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/PlannerContext.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/PlannerContext.java b/fe/src/main/java/org/apache/impala/planner/PlannerContext.java
index 06208aa..b2881b7 100644
--- a/fe/src/main/java/org/apache/impala/planner/PlannerContext.java
+++ b/fe/src/main/java/org/apache/impala/planner/PlannerContext.java
@@ -37,9 +37,6 @@ public class PlannerContext {
// used for determining whether a broadcast join is feasible.
public final static double HASH_TBL_SPACE_OVERHEAD = 1.1;
- // The maximum fraction of remaining memory that a sort node can use during execution.
- public final static double SORT_MEM_MAX_FRACTION = 0.80;
-
// Assumed average number of items in a nested collection, since we currently have no
// statistics on nested fields. The motivation for this constant is to avoid
// pathological plan choices that could result from a SubplanNode having an unknown
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/planner/ValueRange.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/ValueRange.java b/fe/src/main/java/org/apache/impala/planner/ValueRange.java
index 492dd3d..4b8f252 100644
--- a/fe/src/main/java/org/apache/impala/planner/ValueRange.java
+++ b/fe/src/main/java/org/apache/impala/planner/ValueRange.java
@@ -36,7 +36,6 @@ import com.google.common.base.Preconditions;
* value, both bounds are set.
*/
public class ValueRange {
- private final static Logger LOG = LoggerFactory.getLogger(ValueRange.class);
private Expr lowerBound_;
private boolean lowerBoundInclusive_;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/71eb569f/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 5fcd54c..2b6748d 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -112,11 +112,6 @@ public class JniFrontend {
new TBinaryProtocol.Factory();
private final Frontend frontend_;
- // Required minimum value (in milliseconds) for the HDFS config
- // 'dfs.client.file-block-storage-locations.timeout.millis'
- private static final long MIN_DFS_CLIENT_FILE_BLOCK_STORAGE_LOCATIONS_TIMEOUT_MS =
- 10 * 1000;
-
/**
* Create a new instance of the Jni Frontend.
*/
[5/5] incubator-impala git commit: IMPALA-4998: Fix missing table
lock acquisition.
Posted by jr...@apache.org.
IMPALA-4998: Fix missing table lock acquisition.
The following commit broke test_views_compatibility.py
which only runs in exhaustive mode, so the issue was
not caught by pre-commit testing:
a71636847fe742a9d0eb770516aff34ff16bbca1
Testing: Before this patch test_views_compatibility.py
failed locally reliably. After this patch the test
passes locally.
Change-Id: I0e0270daf59fce95f1a1520fc5aaf91d3a7b99fe
Reviewed-on: http://gerrit.cloudera.org:8080/6177
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/c4637960
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/c4637960
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/c4637960
Branch: refs/heads/master
Commit: c46379602af3c007f9f8c98d8d52b8f7dec03270
Parents: 60c1c6e
Author: Alex Behm <al...@cloudera.com>
Authored: Mon Feb 27 21:42:26 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Thu Mar 2 10:31:29 2017 +0000
----------------------------------------------------------------------
.../impala/catalog/CatalogServiceCatalog.java | 61 +++++---------------
.../apache/impala/catalog/TableLoadingMgr.java | 11 +++-
2 files changed, 23 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c4637960/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index 2adba9e..18ece9b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -923,43 +923,17 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
- * Reloads metadata for table 'tbl'. If 'tbl' is an IncompleteTable, it makes an
- * asynchronous request to the table loading manager to create a proper table instance
- * and load the metadata from Hive Metastore. Otherwise, it updates table metadata
- * in-place by calling the load() function on the specified table. Returns the
- * TCatalogObject representing 'tbl', if it is a fully loaded table (e.g. HdfsTable,
- * HBaseTable, etc). Otherwise, returns a newly constructed fully loaded TCatalogObject.
- * Applies proper synchronization to protect the metadata load from concurrent table
- * modifications and assigns a new catalog version.
+ * Reloads metadata for table 'tbl' which must not be an IncompleteTable. Updates the
+ * table metadata in-place by calling load() on the given table. Returns the
+ * TCatalogObject representing 'tbl'. Applies proper synchronization to protect the
+ * metadata load from concurrent table modifications and assigns a new catalog version.
* Throws a CatalogException if there is an error loading table metadata.
*/
public TCatalogObject reloadTable(Table tbl) throws CatalogException {
LOG.info(String.format("Refreshing table metadata: %s", tbl.getFullName()));
- TTableName tblName = new TTableName(tbl.getDb().getName().toLowerCase(),
- tbl.getName().toLowerCase());
- Db db = tbl.getDb();
- if (tbl instanceof IncompleteTable) {
- TableLoadingMgr.LoadRequest loadReq;
- long previousCatalogVersion;
- // Return the table if it is already loaded or submit a new load request.
- catalogLock_.readLock().lock();
- try {
- previousCatalogVersion = tbl.getCatalogVersion();
- loadReq = tableLoadingMgr_.loadAsync(tblName);
- } finally {
- catalogLock_.readLock().unlock();
- }
- Preconditions.checkNotNull(loadReq);
- try {
- // The table may have been dropped/modified while the load was in progress, so
- // only apply the update if the existing table hasn't changed.
- Table result = replaceTableIfUnchanged(loadReq.get(), previousCatalogVersion);
- return result.toTCatalogObject();
- } finally {
- loadReq.close();
- LOG.info(String.format("Refreshed table metadata: %s", tbl.getFullName()));
- }
- }
+ Preconditions.checkState(!(tbl instanceof IncompleteTable));
+ String dbName = tbl.getDb().getName();
+ String tblName = tbl.getName();
if (!tryLockTable(tbl)) {
throw new CatalogException(String.format("Error refreshing metadata for table " +
@@ -971,11 +945,10 @@ public class CatalogServiceCatalog extends Catalog {
try (MetaStoreClient msClient = getMetaStoreClient()) {
org.apache.hadoop.hive.metastore.api.Table msTbl = null;
try {
- msTbl = msClient.getHiveClient().getTable(db.getName(),
- tblName.getTable_name());
+ msTbl = msClient.getHiveClient().getTable(dbName, tblName);
} catch (Exception e) {
throw new TableLoadingException("Error loading metadata for table: " +
- db.getName() + "." + tblName.getTable_name(), e);
+ dbName + "." + tblName, e);
}
tbl.load(true, msClient.getHiveClient(), msTbl);
}
@@ -989,15 +962,6 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
- * Reloads the metadata of a table with name 'tableName'.
- */
- public void reloadTable(TTableName tableName) throws CatalogException {
- Table table = getTable(tableName.getDb_name(), tableName.getTable_name());
- if (table == null) return;
- reloadTable(table);
- }
-
- /**
* Drops the partitions specified in 'partitionSet' from 'tbl'. Throws a
* CatalogException if 'tbl' is not an HdfsTable. Returns the target table.
*/
@@ -1076,7 +1040,12 @@ public class CatalogServiceCatalog extends Catalog {
Table result = removeTable(dbName, tblName);
if (result == null) return null;
tblWasRemoved.setRef(true);
- return result.toTCatalogObject();
+ result.getLock().lock();
+ try {
+ return result.toTCatalogObject();
+ } finally {
+ result.getLock().unlock();
+ }
}
db = getDb(dbName);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/c4637960/fe/src/main/java/org/apache/impala/catalog/TableLoadingMgr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/TableLoadingMgr.java b/fe/src/main/java/org/apache/impala/catalog/TableLoadingMgr.java
index 35cb902..f3abfee 100644
--- a/fe/src/main/java/org/apache/impala/catalog/TableLoadingMgr.java
+++ b/fe/src/main/java/org/apache/impala/catalog/TableLoadingMgr.java
@@ -305,13 +305,18 @@ public class TableLoadingMgr {
}
/**
- * Executes all async refresh work for the specified table name.
+ * Reloads the metadata of the given table to pick up the new cached block location
+ * information. Only reloads the metadata if the table is already loaded. The rationale
+ * is that if the metadata has not been loaded yet, then it needs to be reloaded
+ * anyway, and if the table failed to load, then we do not want to hide errors by
+ * reloading it 'silently' in response to the completion of an HDFS caching request.
*/
private void execAsyncRefreshWork(TTableName tblName) {
if (!waitForCacheDirs(tblName)) return;
try {
- // Reload the table metadata to pickup the new cached block location information.
- catalog_.reloadTable(tblName);
+ Table tbl = catalog_.getTable(tblName.getDb_name(), tblName.getTable_name());
+ if (tbl == null || tbl instanceof IncompleteTable || !tbl.isLoaded()) return;
+ catalog_.reloadTable(tbl);
} catch (CatalogException e) {
LOG.error("Error reloading cached table: ", e);
}