You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2023/09/08 21:53:33 UTC
[impala] 01/02: IMPALA-12424: Allow third party JniFrontend interface.
This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit bc83d46a9a1b6b4a3ca859d3a2835185f4473427
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Wed Sep 6 06:44:05 2023 -0700
IMPALA-12424: Allow third party JniFrontend interface.
This patch allows a third party to inject their own frontend
class instead of using the default JniFrontend included in the
project.
The test case includes an interface that runs queries as normal
except for the "select 1" query which gets changed to "select 42".
Change-Id: I89e677da557b39232847644b6ff17510e2b3c3d5
Reviewed-on: http://gerrit.cloudera.org:8080/20459
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/service/frontend.cc | 8 ++-
bin/start-impala-cluster.py | 7 ++
java/external-frontend/pom.xml | 56 +++++++++++++++
.../apache/impala/external/TestJniFrontend.java | 80 ++++++++++++++++++++++
java/pom.xml | 1 +
tests/custom_cluster/test_external_planner.py | 43 ++++++++++++
6 files changed, 194 insertions(+), 1 deletion(-)
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index 1d1fe51fc..4dcfd2506 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -84,6 +84,12 @@ DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The
"optional.");
DEFINE_bool(enable_kudu_impala_hms_check, true, "By default this flag is true. If "
"enabled checks that Kudu and Impala are using the same HMS instance(s).");
+DEFINE_string(jni_frontend_class, "org/apache/impala/service/JniFrontend", "By default "
+ "the JniFrontend class included in the repository is used as the frontend interface. "
+ "This option allows the class to be overridden by a third party module. The "
+ "overridden class needs to contain all the methods in the methods[] variable, so "
+ "most implementations should make their class a child of JniFrontend and "
+ "override only relevant methods.");
Frontend::Frontend() {
JniMethodDescriptor methods[] = {
@@ -133,7 +139,7 @@ Frontend::Frontend() {
ABORT_IF_ERROR(jni_frame.push(jni_env));
// create instance of java class JniFrontend
- jclass fe_class = jni_env->FindClass("org/apache/impala/service/JniFrontend");
+ jclass fe_class = jni_env->FindClass(FLAGS_jni_frontend_class.c_str());
ABORT_IF_EXC(jni_env);
uint32_t num_methods = sizeof(methods) / sizeof(methods[0]);
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 0f0c0e216..b2a5a48b4 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -155,6 +155,9 @@ parser.add_option("--enable_catalogd_ha", dest="enable_catalogd_ha",
action="store_true", default=False,
help="If true, enables CatalogD HA - the cluster will be launched "
"with two catalogd instances as Active-Passive HA pair.")
+parser.add_option("--jni_frontend_class", dest="jni_frontend_class",
+ action="store", default="org/apache/impala/service/JniFrontend",
+ help="Use a custom java frontend interface.")
# For testing: list of comma-separated delays, in milliseconds, that delay impalad catalog
# replica initialization. The ith delay is applied to the ith impalad.
@@ -485,6 +488,10 @@ def build_impalad_arg_lists(cluster_size, num_coordinators, use_exclusive_coordi
args = "{args} -geospatial_library={geospatial_library}".format(
args=args, geospatial_library=options.geospatial_library)
+ if options.jni_frontend_class:
+ args = "-jni_frontend_class={jni_frontend_class} {args}".format(
+ jni_frontend_class=options.jni_frontend_class, args=args)
+
# Appended at the end so they can override previous args.
if i < len(per_impalad_args):
args = "{args} {per_impalad_args}".format(
diff --git a/java/external-frontend/pom.xml b/java/external-frontend/pom.xml
new file mode 100644
index 000000000..c29fd78fd
--- /dev/null
+++ b/java/external-frontend/pom.xml
@@ -0,0 +1,56 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-parent</artifactId>
+ <version>4.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>external-frontend</artifactId>
+ <version>1.0</version>
+ <packaging>jar</packaging>
+
+ <name>external-frontend</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.impala</groupId>
+ <artifactId>impala-frontend</artifactId>
+ <version>4.3.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.11.0</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/java/external-frontend/src/main/java/org/apache/impala/external/TestJniFrontend.java b/java/external-frontend/src/main/java/org/apache/impala/external/TestJniFrontend.java
new file mode 100644
index 000000000..0c4ef6b05
--- /dev/null
+++ b/java/external-frontend/src/main/java/org/apache/impala/external/TestJniFrontend.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.impala.externalfrontend;
+
+import com.google.common.base.Preconditions;
+import org.apache.impala.common.ImpalaException;
+import org.apache.impala.common.InternalException;
+import org.apache.impala.common.JniUtil;
+import org.apache.impala.authorization.AuthorizationFactory;
+import org.apache.impala.service.BackendConfig;
+import org.apache.impala.service.Frontend;
+import org.apache.impala.service.Frontend.PlanCtx;
+import org.apache.impala.service.JniFrontend;
+import org.apache.impala.thrift.TExecRequest;
+import org.apache.impala.thrift.TQueryCtx;
+import org.apache.impala.util.AuthorizationUtil;
+import org.apache.thrift.TException;
+
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.TSerializer;
+
+public class TestJniFrontend extends JniFrontend {
+ private final static TBinaryProtocol.Factory protocolFactory_ =
+ new TBinaryProtocol.Factory();
+
+ private final Frontend testFrontend_;
+
+ public TestJniFrontend(byte[] thriftBackendConfig, boolean isBackendTest)
+ throws ImpalaException, TException {
+ super(thriftBackendConfig, isBackendTest);
+ final AuthorizationFactory authzFactory =
+ AuthorizationUtil.authzFactoryFrom(BackendConfig.INSTANCE);
+ testFrontend_ = new Frontend(authzFactory, false);
+ }
+
+ /**
+ * Jni wrapper for Frontend.createExecRequest(). Accepts a serialized
+ * TQueryContext; returns a serialized TQueryExecRequest.
+ * This specific version will execute normally with the exception of the
+ * 'select 1' query which will instead implement a 'select 42' query, which
+ * is so bizarre, that it will prove that we hit this code. '42' also answers
+ * the question, "What is the meaning of life?"
+ */
+ @Override
+ public byte[] createExecRequest(byte[] thriftQueryContext)
+ throws ImpalaException {
+ Preconditions.checkNotNull(testFrontend_);
+ TQueryCtx queryCtx = new TQueryCtx();
+ JniUtil.deserializeThrift(protocolFactory_, queryCtx, thriftQueryContext);
+ if (queryCtx.getClient_request().getStmt().equals("select 1")) {
+ queryCtx.getClient_request().setStmt("select 42");
+ }
+
+ PlanCtx planCtx = new PlanCtx(queryCtx);
+ TExecRequest result = testFrontend_.createExecRequest(planCtx);
+
+ try {
+ TSerializer serializer = new TSerializer(protocolFactory_);
+ return serializer.serialize(result);
+ } catch (TException e) {
+ throw new InternalException(e.getMessage());
+ }
+ }
+}
diff --git a/java/pom.xml b/java/pom.xml
index 76364f06d..b5d6d0250 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -368,6 +368,7 @@ under the License.
<module>executor-deps</module>
<module>ext-data-source</module>
<module>../fe</module>
+ <module>external-frontend</module>
<module>query-event-hook-api</module>
<module>shaded-deps/hive-exec</module>
<module>shaded-deps/s3a-aws-sdk</module>
diff --git a/tests/custom_cluster/test_external_planner.py b/tests/custom_cluster/test_external_planner.py
new file mode 100644
index 000000000..17155ab74
--- /dev/null
+++ b/tests/custom_cluster/test_external_planner.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function
+import logging
+import pytest
+import os
+
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+
+LOG = logging.getLogger(__name__)
+
+
+class TestExternalPlanner(CustomClusterTestSuite):
+
+ @classmethod
+ def setup_class(cls):
+ super(TestExternalPlanner, cls).setup_class()
+ os.environ["CUSTOM_CLASSPATH"] = os.getenv("IMPALA_HOME") + \
+ "/java/external-frontend/target/external-frontend-1.0.jar"
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="-jni_frontend_class=org/apache/impala/externalfrontend/TestJniFrontend")
+ def test_external_frontend(self, vector):
+ setup_client = self.create_impala_client()
+ assert setup_client.execute("select 2").data == ['2']
+ # The custom JniFrontend overrides the 'select 1' query with a 'select 42' query.
+ assert setup_client.execute("select 1").data == ['42']