You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bo...@apache.org on 2026/02/12 10:03:04 UTC
(impala) branch master updated (6302cdadd -> 26e3529c9)
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 6302cdadd IMPALA-14665: Upgrade Iceberg to 1.10.1
new ffcb9a9ec IMPALA-14739: Log if local catalog cache entry size is truncated
new 6b4c895f9 IMPALA-14719: Fixed race condition for Calcite property
new 26e3529c9 IMPALA-14734: Optimize sorting file descriptors during planning
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/impala/catalog/FileDescriptor.java | 28 ++++++++++++-----
.../impala/catalog/local/CatalogdMetaProvider.java | 6 +++-
.../org/apache/impala/planner/IcebergScanNode.java | 3 +-
.../java/org/apache/impala/service/Frontend.java | 36 +++++++++++++---------
.../calcite/service/CalciteCompilerFactory.java | 4 ---
5 files changed, 50 insertions(+), 27 deletions(-)
(impala) 01/03: IMPALA-14739: Log if local catalog cache entry size is truncated
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ffcb9a9ec74b9797e2e1b231f3aa1ca074da35a4
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Wed Feb 11 19:36:30 2026 +0100
IMPALA-14739: Log if local catalog cache entry size is truncated
The weigh() function cannot return more than 2GB. Such huge objects
should be very rare, but if it happens, the cache will underestimate
its size. Add a warning in this case.
Change-Id: I30554e55e0d487be38e10ddadb7fe53410f61201
Reviewed-on: http://gerrit.cloudera.org:8080/23964
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/catalog/local/CatalogdMetaProvider.java | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 7db96fdcc..7eccfd198 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -208,7 +208,7 @@ import com.google.errorprone.annotations.concurrent.GuardedBy;
*/
public class CatalogdMetaProvider implements MetaProvider {
- private final static Logger LOG = LoggerFactory.getLogger(CatalogdMetaProvider.class);
+ private static final Logger LOG = LoggerFactory.getLogger(CatalogdMetaProvider.class);
/**
* Sentinel value used as a negative cache entry for column statistics.
@@ -2293,6 +2293,9 @@ public class CatalogdMetaProvider implements MetaProvider {
@VisibleForTesting
static class SizeOfWeigher implements Weigher<Object, Object> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SizeOfWeigher.class);
+
// Bypass flyweight objects like small boxed integers, Boolean.TRUE, enums, etc.
private static final boolean BYPASS_FLYWEIGHT = true;
// Cache the reflected sizes of classes seen.
@@ -2351,6 +2354,7 @@ public class CatalogdMetaProvider implements MetaProvider {
entrySize_.update(size);
}
if (size > Integer.MAX_VALUE) {
+ LOG.warn("weight exceeded Integer.MAX_VALUE: {}", size);
return Integer.MAX_VALUE;
}
return (int)size;
(impala) 02/03: IMPALA-14719: Fixed race condition for Calcite property
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6b4c895f9aa1d3989c343f05270bc6a60815bc45
Author: Steve Carlin <sc...@cloudera.com>
AuthorDate: Tue Feb 3 09:55:11 2026 -0800
IMPALA-14719: Fixed race condition for Calcite property
IMPALA-14710 was filed for a flaky test involving the
calcite.default.charset property. Setting this property
before the jar file is loaded fixes this issue.
This commit also fixes two other issues:
- IMPALA-14384: Class.forName() is only called once now. The compiler
factory is stored in a singleton.
- The Calcite planner is now part of the build and should be in the
target directory. If the user explicitly sets the calcite planner
option and the calcite jar file not found, an error is thrown rather
than falling back.
Change-Id: I80a0207150cf521c297b12a24075ae5f29c5c5ff
Reviewed-on: http://gerrit.cloudera.org:8080/23935
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/service/Frontend.java | 36 +++++++++++++---------
.../calcite/service/CalciteCompilerFactory.java | 4 ---
2 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index f9f8184f4..35ddee177 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -308,6 +308,10 @@ public class Frontend {
public static final String PLANNER_PROFILE = "PlannerInfo";
public static final String PLANNER_TYPE = "PlannerType";
+ // The Calcite compiler is loaded on demand, not linked in. The factory is a
+ // singleton.
+ private static CompilerFactory calciteCompilerFactory = loadCalciteCompilerFactory();
+
/**
* Plan-time context that allows capturing various artifacts created
* during the process.
@@ -3777,27 +3781,31 @@ public class Frontend {
}
@Nullable
- private CompilerFactory getCalciteCompilerFactory(PlanCtx ctx) {
+ private CompilerFactory getCalciteCompilerFactory(PlanCtx ctx)
+ throws ImpalaException {
TQueryOptions queryOptions = ctx.getQueryContext().client_request.getQuery_options();
- LOG.info("Searching for planner to use...");
if (queryOptions.isUse_calcite_planner()) {
- try {
- CompilerFactory calciteFactory = (CompilerFactory) Class.forName(
- "org.apache.impala.calcite.service." +
- "CalciteCompilerFactory").newInstance();
- if (calciteFactory != null) {
- LOG.info("Found Calcite Planner, using it.");
- return calciteFactory;
- } else {
- LOG.info("Could not find Calcite planner, using original planner.");
- }
- } catch (Exception e) {
- LOG.info("Could not find Calcite planner, using original planner: " + e);
+ if (calciteCompilerFactory == null) {
+ throw new InternalException("Could not find Calcite planner");
}
+ return calciteCompilerFactory;
}
return null;
}
+ private static CompilerFactory loadCalciteCompilerFactory() {
+ LOG.info("Loading Calcite Planner jar file");
+ try {
+ System.setProperty("calcite.default.charset", "UTF8");
+ return (CompilerFactory) Class.forName(
+ "org.apache.impala.calcite.service." +
+ "CalciteCompilerFactory").newInstance();
+ } catch (Exception e) {
+ LOG.info("Could not find Calcite planner: " + e);
+ return null;
+ }
+ }
+
public String getShowCreateTable(
TTableName tname, boolean withStats, int partitionLimit) throws ImpalaException {
Frontend.RetryTracker retries = new Frontend.RetryTracker(
diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteCompilerFactory.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteCompilerFactory.java
index 98e518d9a..e2eb59319 100644
--- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteCompilerFactory.java
+++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteCompilerFactory.java
@@ -45,10 +45,6 @@ public class CalciteCompilerFactory implements CompilerFactory {
protected static final Logger LOG =
LoggerFactory.getLogger(CalciteCompilerFactory.class.getName());
- static {
- System.setProperty("calcite.default.charset", "UTF8");
- }
-
private static final String PLANNER = "CalcitePlanner";
static {
(impala) 03/03: IMPALA-14734: Optimize sorting file descriptors during planning
Posted by bo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
boroknagyz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 26e3529c95dd63e50e9f59a9871084dccb28d868
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Tue Feb 10 20:58:25 2026 +0100
IMPALA-14734: Optimize sorting file descriptors during planning
IcebergScanNode sorts the file descriptors by path (IMPALA-12765).
This can dominate planning time if there are many files.
This change makes this faster by avoiding extracting Java
Strings from flatbuffer, which involves utf8 decoding. Also
changes a few similar functions to avoid duplicate decoding.
For a table with ~1 million files:
explain select * from bigice limit 1;
before: ~12s
after: ~6.5s
Change-Id: Icb914eb4de7bdadeb876f7dd101e8737b9527b6f
Reviewed-on: http://gerrit.cloudera.org:8080/23958
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Csaba Ringhofer <cs...@cloudera.com>
---
.../org/apache/impala/catalog/FileDescriptor.java | 28 ++++++++++++++++------
.../org/apache/impala/planner/IcebergScanNode.java | 3 ++-
2 files changed, 23 insertions(+), 8 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FileDescriptor.java b/fe/src/main/java/org/apache/impala/catalog/FileDescriptor.java
index 16b3009e7..3bba32a82 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FileDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FileDescriptor.java
@@ -215,9 +215,8 @@ public class FileDescriptor implements Comparable<FileDescriptor> {
}
public String getAbsolutePath() {
- return StringUtils.isEmpty(fbFileDescriptor_.absolutePath()) ?
- StringUtils.EMPTY :
- fbFileDescriptor_.absolutePath();
+ String path = fbFileDescriptor_.absolutePath();
+ return StringUtils.isEmpty(path) ? StringUtils.EMPTY : path;
}
public String getAbsolutePath(String rootPath) {
@@ -230,14 +229,21 @@ public class FileDescriptor implements Comparable<FileDescriptor> {
}
public String getPath() {
- if (StringUtils.isEmpty(fbFileDescriptor_.relativePath())
- && StringUtils.isNotEmpty(fbFileDescriptor_.absolutePath())) {
- return fbFileDescriptor_.absolutePath();
+ String relativePath = fbFileDescriptor_.relativePath();
+ if (StringUtils.isEmpty(relativePath)) {
+ String absolutePath = fbFileDescriptor_.absolutePath();
+ return StringUtils.isEmpty(absolutePath) ? StringUtils.EMPTY : absolutePath;
} else {
- return fbFileDescriptor_.relativePath();
+ return relativePath;
}
}
+ public ByteBuffer getPathAsByteBuffer() {
+ ByteBuffer relativePath = fbFileDescriptor_.relativePathAsByteBuffer();
+ if (relativePath != null) return relativePath;
+ return fbFileDescriptor_.absolutePathAsByteBuffer();
+ }
+
public long getFileLength() {
return fbFileDescriptor_.length();
}
@@ -305,6 +311,14 @@ public class FileDescriptor implements Comparable<FileDescriptor> {
return getPath().compareTo(otherFd.getPath());
}
+ /**
+ * Does lexical comparison without UTF8 decoding. Faster alternative to compareTo where
+ * the exact order is not critical.
+ */
+ public int byteBufferCompareTo(FileDescriptor otherFd) {
+ return getPathAsByteBuffer().compareTo(otherFd.getPathAsByteBuffer());
+ }
+
/**
* Compares the modification time and file size between current FileDescriptor and the
* latest FileStatus to determine if the file has changed. Returns true if the file
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
index 58b1a3dbb..ed8e5808c 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
@@ -113,7 +113,8 @@ public class IcebergScanNode extends HdfsScanNode {
// Create a clone of the original file descriptor list to avoid getting
// ConcurrentModificationException when sorting.
fileDescs_ = new ArrayList<>(fileDescs_);
- Collections.sort(fileDescs_);
+ Collections.sort(fileDescs_,
+ (IcebergFileDescriptor a, IcebergFileDescriptor b) -> a.byteBufferCompareTo(b));
filesAreSorted_ = true;
}
nonIdentityConjuncts_ = nonIdentityConjuncts;