You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2022/02/09 19:01:26 UTC
[impala] branch master updated (85c13b3 -> 8ddac48)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 85c13b3 IMPALA-11050: Skip filemetadata reloading in processing AlterPartition event from event processor
new 4bcc29f IMPALA-11109: Catch class loading error for UDFs.
new 8ddac48 IMPALA-10989: fix race for result set metadata
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/service/client-request-state.cc | 8 ++++--
.../hive/executor/HiveLegacyJavaFunction.java | 33 ++++++++++++++--------
.../hive/executor/HiveLegacyJavaFunctionTest.java | 4 +--
3 files changed, 29 insertions(+), 16 deletions(-)
[impala] 01/02: IMPALA-11109: Catch class loading error for UDFs.
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 4bcc29f256acf751d168aacc0c3b14bcbfed5c10
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Sat Feb 5 09:03:50 2022 -0800
IMPALA-11109: Catch class loading error for UDFs.
Fixes regression caused by IMPALA-10997.
If a UDF fails to load, an exception needs to be caught
and logged or else catalogd will not start up. The
method HiveLegacyJavaFunction.extract() will catch any hidden
exception thrown, and rethrow it as a CatalogException. The caller
already handles the CatalogException and logs the message
appropriately. The specific "hidden" exception thrown in our
regression tests was a "ClassNotFoundException", but in case
there are other exceptions that are thrown, we catch the generic
"Throwable" exception because the "extract" method should not
prevent the server from coming up.
Change-Id: I16813209cd4c2367c45e569c2aac13eff7ce2dbf
Reviewed-on: http://gerrit.cloudera.org:8080/18207
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
---
.../hive/executor/HiveLegacyJavaFunction.java | 33 ++++++++++++++--------
.../hive/executor/HiveLegacyJavaFunctionTest.java | 4 +--
2 files changed, 23 insertions(+), 14 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/hive/executor/HiveLegacyJavaFunction.java b/fe/src/main/java/org/apache/impala/hive/executor/HiveLegacyJavaFunction.java
index 6b5773b..e88f166 100644
--- a/fe/src/main/java/org/apache/impala/hive/executor/HiveLegacyJavaFunction.java
+++ b/fe/src/main/java/org/apache/impala/hive/executor/HiveLegacyJavaFunction.java
@@ -147,21 +147,30 @@ public class HiveLegacyJavaFunction implements HiveJavaFunction {
String jarUri = hiveFn_.getResourceUris().get(0).getUri();
// Load each method in the UDF class and create the corresponding Impala Function
// object.
- for (Method m: UDF_.getClass().getMethods()) {
- if (m.getName().equals(UDF_FUNCTION_NAME)) {
- ScalarFunction fn = fromHiveFunction(hiveFn_.getDbName(),
- hiveFn_.getFunctionName(), hiveFn_.getClassName(),
- m.getParameterTypes(), m.getReturnType(), jarUri);
- if (fn != null) {
- if (!addedSignatures.contains(fn.signatureString())) {
- result.add(fn);
- addedSignatures.add(fn.signatureString());
+ try {
+ for (Method m: UDF_.getClass().getMethods()) {
+ if (m.getName().equals(UDF_FUNCTION_NAME)) {
+ ScalarFunction fn = fromHiveFunction(hiveFn_.getDbName(),
+ hiveFn_.getFunctionName(), hiveFn_.getClassName(),
+ m.getParameterTypes(), m.getReturnType(), jarUri);
+ if (fn != null) {
+ if (!addedSignatures.contains(fn.signatureString())) {
+ result.add(fn);
+ addedSignatures.add(fn.signatureString());
+ }
+ } else {
+ LOG.warn("Ignoring incompatible method: " + m.toString() + " during load of "
+ + "Hive UDF:" + hiveFn_.getFunctionName() + " from " + UDF_.getClass());
}
- } else {
- LOG.warn("Ignoring incompatible method: " + m.toString() + " during load of "
- + "Hive UDF:" + hiveFn_.getFunctionName() + " from " + UDF_.getClass());
}
}
+ } catch (Throwable t) {
+ // Catch all runtime exceptions here. One possible runtime exception that can occur
+ // is ClassNotFoundException thrown by UDF_.getClass(). We want to catch all
+ // possible exceptions, mark it as a CatalogException, and let the caller decide on
+ // how to handle it.
+ throw new CatalogException("Error loading function " + hiveFn_.getFunctionName() +
+ ": " + t);
}
if (result.isEmpty()) {
throw new CatalogException("No compatible function signatures found.");
diff --git a/fe/src/test/java/org/apache/impala/hive/executor/HiveLegacyJavaFunctionTest.java b/fe/src/test/java/org/apache/impala/hive/executor/HiveLegacyJavaFunctionTest.java
index 4aaa88f..4341094 100644
--- a/fe/src/test/java/org/apache/impala/hive/executor/HiveLegacyJavaFunctionTest.java
+++ b/fe/src/test/java/org/apache/impala/hive/executor/HiveLegacyJavaFunctionTest.java
@@ -147,8 +147,8 @@ public class HiveLegacyJavaFunctionTest {
try {
testScalar(udfClass, expectedFuncs);
} catch (CatalogException e) {
- Assert.assertEquals(e.getMessage(),
- "No compatible function signatures found.");
+ Assert.assertTrue(e.getMessage().contains(
+ "No compatible function signatures found."));
return;
}
Assert.fail("Extraction should not have succeeded.");
[impala] 02/02: IMPALA-10989: fix race for result set metadata
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8ddac48f3428c86f2cbd037ced89cfb903298b12
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Mon Feb 7 15:27:35 2022 -0800
IMPALA-10989: fix race for result set metadata
TSAN tests uncovered a race condition between the
thread reading the result set metadata in
ImpalaServer::GetResultSetMetadata() and the
thread setting the result set metadata in
ClientRequestState::SetResultSet() from
ClientRequestState::ExecDdlRequestImpl().
This is introduced by IMPALA-10811, which runs
ExecDdlRequestImpl in an async thread that
can now race with the client thread.
GetResultSetMetadata() holds ClientRequestState's
lock_ while reading the result set metadata, so
the fix is to hold this lock when writing the
result set metadata.
Testing:
- Ran TSAN core job
Change-Id: Ic0833ed20d62474c434fa94bbbf8cd8ea99a7cf4
Reviewed-on: http://gerrit.cloudera.org:8080/18212
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
---
be/src/service/client-request-state.cc | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/be/src/service/client-request-state.cc b/be/src/service/client-request-state.cc
index ca0f6c0..af68245 100644
--- a/be/src/service/client-request-state.cc
+++ b/be/src/service/client-request-state.cc
@@ -721,8 +721,12 @@ void ClientRequestState::ExecDdlRequestImpl(bool exec_in_worker_thread) {
ExecQueryOrDmlRequest(exec_request_->query_exec_request, !exec_in_worker_thread));
}
- // Set the results to be reported to the client.
- SetResultSet(catalog_op_executor_->ddl_exec_response());
+ // Set the results to be reported to the client. Do this under lock to avoid races
+ // with ImpalaServer::GetResultSetMetadata().
+ {
+ lock_guard<mutex> l(lock_);
+ SetResultSet(catalog_op_executor_->ddl_exec_response());
+ }
}
bool ClientRequestState::ShouldRunExecDdlAsync() {