You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by la...@apache.org on 2025/03/28 13:43:56 UTC
(impala) branch master updated (7f38c7ed6 -> e6078b428)
This is an automated email from the ASF dual-hosted git repository.
laszlog pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 7f38c7ed6 IMPALA-13890: Deflake test_coord_only_pool_one_quiescing_coord
new 0b3bc412f IMPALA-13738 (Part1): Refactor IcebergTable.getPartialInfo()
new 2cf577789 IMPALA-13826: Migrate from imp to importlib in the config generator
new e6078b428 IMPALA-13825: Extend Docker container build to custom base images
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/generate_xml_config.py | 30 +++++++-
bin/impala-config.sh | 11 +++
.../dockerized-impala-bootstrap-and-test.sh | 3 +-
bin/start-impala-cluster.py | 60 ++++++++++++++++
docker/CMakeLists.txt | 69 ++++++++++--------
docker/daemon_entrypoint.sh | 22 +++++-
docker/docker-build.sh | 1 +
docker/impala_base/Dockerfile | 10 ++-
docker/impala_profile_tool/Dockerfile | 13 +++-
docker/install_bash_if_needed.sh | 74 ++++++++++++++++++++
docker/install_os_packages.sh | 65 ++++++++++++++++-
docker/setup_build_context.py | 5 ++
.../org/apache/impala/catalog/FeFsPartition.java | 42 +++++++++++
.../java/org/apache/impala/catalog/FeFsTable.java | 7 ++
.../org/apache/impala/catalog/HdfsPartition.java | 81 ++++++++++++++++++----
.../java/org/apache/impala/catalog/HdfsTable.java | 75 ++++----------------
.../org/apache/impala/catalog/IcebergTable.java | 73 +++++++++++++++----
.../apache/impala/catalog/local/LocalFsTable.java | 5 ++
tests/common/impala_connection.py | 4 +-
19 files changed, 520 insertions(+), 130 deletions(-)
create mode 100755 docker/install_bash_if_needed.sh
(impala) 01/03: IMPALA-13738 (Part1): Refactor IcebergTable.getPartialInfo()
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
laszlog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0b3bc412f2b09f4818cb0c50fccec5cfb3ddfe8d
Author: Noemi Pap-Takacs <np...@cloudera.com>
AuthorDate: Wed Mar 5 16:45:19 2025 +0100
IMPALA-13738 (Part1): Refactor IcebergTable.getPartialInfo()
This patch aims to refactor Iceberg tables in the Catalog.
It reduces the dependency on the internal hdfsTable_ object in
IcebergTable.getPartialInfo(). Now we depend less on HdfsTable
when we load an Iceberg table in local catalog mode.
The remaining dependencies will be addressed in following patches.
Now creating TPartialPartitionInfo is the responsibility of the
partitions, not the tables:
- FeFsPartition holds a basic default implementation for the
unpartitioned tables, which includes all Iceberg tables (we
handle Iceberg tables similarly to unpartitioned HDFS tables
in the Catalog).
- HdfsPartition extends this implementation with HDFS and
Hive ACID specific fields.
Testing:
- existing Iceberg e2e tests in local catalog mode
- existing Hive ACID tests in local catalog mode
- ran test_iceberg_with_puffin.py in local catalog mode
- LocalCatalog FE test
Change-Id: I75d277fb555d0088a2e28111ff529a4605d734fa
Reviewed-on: http://gerrit.cloudera.org:8080/22605
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/catalog/FeFsPartition.java | 42 +++++++++++
.../java/org/apache/impala/catalog/FeFsTable.java | 7 ++
.../org/apache/impala/catalog/HdfsPartition.java | 81 ++++++++++++++++++----
.../java/org/apache/impala/catalog/HdfsTable.java | 75 ++++----------------
.../org/apache/impala/catalog/IcebergTable.java | 73 +++++++++++++++----
.../apache/impala/catalog/local/LocalFsTable.java | 5 ++
6 files changed, 194 insertions(+), 89 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeFsPartition.java b/fe/src/main/java/org/apache/impala/catalog/FeFsPartition.java
index bcec2a634..40fba33c1 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeFsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeFsPartition.java
@@ -17,6 +17,7 @@
package org.apache.impala.catalog;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,8 +29,10 @@ import org.apache.hadoop.fs.Path;
import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.thrift.TAccessLevel;
+import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
import org.apache.impala.thrift.THdfsPartitionLocation;
import org.apache.impala.thrift.TNetworkAddress;
+import org.apache.impala.thrift.TPartialPartitionInfo;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.util.ListMap;
@@ -209,4 +212,43 @@ public interface FeFsPartition {
* Returns new FeFsPartition that has the delete delta descriptors as file descriptors.
*/
FeFsPartition genDeleteDeltaPartition();
+
+ /**
+ * Creates TPartialPartitionInfo for the default partition (the one and only partition
+ * that unpartitioned tables have).
+ */
+ default TPartialPartitionInfo getDefaultPartialPartitionInfo(
+ TGetPartialCatalogObjectRequest req) {
+
+ TPartialPartitionInfo partInfo = new TPartialPartitionInfo(getId());
+
+ if (req.table_info_selector.want_partition_names) {
+ partInfo.setName(getPartitionName());
+ }
+
+ if (req.table_info_selector.want_partition_metadata) {
+ // We do not need to set partition metadata for unpartitioned tables.
+ partInfo.setHas_incremental_stats(hasIncrementalStats());
+ }
+
+ if (req.table_info_selector.want_partition_files) {
+ partInfo.setLast_compaction_id(-1);
+ partInfo.insert_file_descriptors = new ArrayList<>();
+ partInfo.delete_file_descriptors = new ArrayList<>();
+ partInfo.file_descriptors = new ArrayList<>();
+ if (!getTable().isHiveAcid()) {
+ for (FileDescriptor fd: getFileDescriptors()) {
+ partInfo.file_descriptors.add(fd.toThrift());
+ }
+ }
+ }
+
+ if (req.table_info_selector.want_partition_stats) {
+ partInfo.setPartition_stats(getPartitionStatsCompressed());
+ }
+
+ partInfo.setIs_marked_cached(isMarkedCached());
+
+ return partInfo;
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
index b4d464b4f..5d66c8e74 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeFsTable.java
@@ -277,6 +277,13 @@ public interface FeFsTable extends FeTable {
*/
SqlConstraints getSqlConstraints();
+ /**
+ * @return whether it is a Hive ACID table.
+ */
+ default boolean isHiveAcid() {
+ return false;
+ }
+
default FileSystem getFileSystem() throws CatalogException {
FileSystem tableFs;
try {
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
index c3eef49fe..7cece6268 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
@@ -40,6 +40,7 @@ import javax.annotation.Nullable;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.common.ValidWriteIdList;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.LiteralExpr;
@@ -58,12 +59,14 @@ import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TExpr;
import org.apache.impala.thrift.TExprNode;
+import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
import org.apache.impala.thrift.THdfsFileDesc;
import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.THdfsPartitionLocation;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TPartialPartitionInfo;
import org.apache.impala.thrift.TPartitionStats;
+import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.HdfsCachingUtil;
import org.apache.impala.util.ListMap;
import org.slf4j.Logger;
@@ -592,6 +595,70 @@ public class HdfsPartition extends CatalogObjectImpl
encodedDeleteFileDescriptors_.size();
}
+ /**
+ * Returns the requested partitions info and the number of files filtered out based on
+ * the ACID writeIdList.
+ * For non-ACID tables this number is always 0. This number being null indicates that
+ * the file descriptors are missing from the Catalog.
+ */
+ public Pair<TPartialPartitionInfo, Integer> getPartialPartitionInfo(
+ TGetPartialCatalogObjectRequest req, ValidWriteIdList reqWriteIdList) {
+
+ TPartialPartitionInfo partInfo =
+ FeFsPartition.super.getDefaultPartialPartitionInfo(req);
+ // Add HdfsPartition specific information to the basic default partition info.
+ if (req.table_info_selector.want_hms_partition) {
+ partInfo.hms_partition = toHmsPartition();
+ }
+ // The special "prototype partition" or the only partition of an unpartitioned table
+ // don't have partition metadata.
+ if (req.table_info_selector.want_partition_metadata
+ && table_.isPartitioned()
+ && id_ != CatalogObjectsConstants.PROTOTYPE_PARTITION_ID) {
+ // Don't need to make a copy here since the HMS parameters shouldn't change during
+ // the invocation of getPartialPartitionInfo().
+ partInfo.hms_parameters = getParameters();
+ partInfo.write_id = writeId_;
+ partInfo.hdfs_storage_descriptor = fileFormatDescriptor_.toThrift();
+ partInfo.location = getLocationAsThrift();
+ }
+ int numFilesFiltered = 0;
+ if (req.table_info_selector.want_partition_files) {
+ partInfo.setLast_compaction_id(getLastCompactionId());
+ if (table_.isHiveAcid()) {
+ try {
+ if (!getInsertFileDescriptors().isEmpty()) {
+ numFilesFiltered += addFilteredFds(getInsertFileDescriptors(),
+ partInfo.insert_file_descriptors, reqWriteIdList);
+ numFilesFiltered += addFilteredFds(getDeleteFileDescriptors(),
+ partInfo.delete_file_descriptors, reqWriteIdList);
+ } else {
+ numFilesFiltered += addFilteredFds(getFileDescriptors(),
+ partInfo.file_descriptors, reqWriteIdList);
+ }
+ } catch (CatalogException ex) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not use cached file descriptors of partition {} of table {}"
+ + " for writeIdList {}", getPartitionName(), getTable().getFullName(),
+ reqWriteIdList, ex);
+ }
+ return new Pair<>(partInfo, null);
+ }
+ }
+ }
+ return new Pair<>(partInfo, numFilesFiltered);
+ }
+
+ private int addFilteredFds(List<FileDescriptor> fds, List<THdfsFileDesc> thriftFds,
+ ValidWriteIdList writeIdList) throws CatalogException {
+ List<FileDescriptor> filteredFds = new ArrayList<>(fds);
+ int numFilesFiltered = AcidUtils.filterFdsForAcidState(filteredFds, writeIdList);
+ for (FileDescriptor fd: filteredFds) {
+ thriftFds.add(fd.toThrift());
+ }
+ return numFilesFiltered;
+ }
+
public FileMetadataStats getFileMetadataStats() {
return fileMetadataStats_;
}
@@ -606,20 +673,6 @@ public class HdfsPartition extends CatalogObjectImpl
return cachedMsPartitionDescriptor_;
}
- public void setPartitionMetadata(TPartialPartitionInfo tPart) {
- // The special "prototype partition" or the only partition of an unpartitioned table
- // don't have partition metadata.
- if (id_ == CatalogObjectsConstants.PROTOTYPE_PARTITION_ID
- || !table_.isPartitioned()) {
- return;
- }
- // Don't need to make a copy here since the caller should not modify the parameters.
- tPart.hms_parameters = getParameters();
- tPart.write_id = writeId_;
- tPart.hdfs_storage_descriptor = fileFormatDescriptor_.toThrift();
- tPart.location = getLocationAsThrift();
- }
-
/**
* Returns a Hive-compatible partition object that may be used in calls to the
* metastore.
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 020df5e1d..00994419b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -75,7 +75,6 @@ import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TColumn;
import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
import org.apache.impala.thrift.TGetPartialCatalogObjectResponse;
-import org.apache.impala.thrift.THdfsFileDesc;
import org.apache.impala.thrift.THdfsPartition;
import org.apache.impala.thrift.THdfsTable;
import org.apache.impala.thrift.TNetworkAddress;
@@ -461,6 +460,12 @@ public class HdfsTable extends Table implements FeFsTable {
return null;
}
+ @Override
+ public boolean isHiveAcid() {
+ if (validWriteIds_ != null) return true;
+ return AcidUtils.isTransactionalTable(msTable_.getParameters());
+ }
+
/**
* Marks a partition dirty by registering the partition builder for its new instance.
*/
@@ -2293,56 +2298,16 @@ public class HdfsTable extends Table implements FeFsTable {
return new TGetPartialCatalogObjectResponse().setLookup_status(
CatalogLookupStatus.PARTITION_NOT_FOUND);
}
- TPartialPartitionInfo partInfo = new TPartialPartitionInfo(partId);
-
- if (req.table_info_selector.want_partition_names) {
- partInfo.setName(part.getPartitionName());
- }
-
- if (req.table_info_selector.want_partition_metadata) {
- part.setPartitionMetadata(partInfo);
- partInfo.setHas_incremental_stats(part.hasIncrementalStats());
- }
- if (req.table_info_selector.want_hms_partition) {
- partInfo.hms_partition = part.toHmsPartition();
- }
-
- if (req.table_info_selector.want_partition_files) {
- partInfo.setLast_compaction_id(part.getLastCompactionId());
- try {
- if (!part.getInsertFileDescriptors().isEmpty()) {
- partInfo.file_descriptors = new ArrayList<>();
- partInfo.insert_file_descriptors = new ArrayList<>();
- numFilesFiltered += addFilteredFds(part.getInsertFileDescriptors(),
- partInfo.insert_file_descriptors, reqWriteIdList);
- partInfo.delete_file_descriptors = new ArrayList<>();
- numFilesFiltered += addFilteredFds(part.getDeleteFileDescriptors(),
- partInfo.delete_file_descriptors, reqWriteIdList);
- } else {
- partInfo.file_descriptors = new ArrayList<>();
- numFilesFiltered += addFilteredFds(part.getFileDescriptors(),
- partInfo.file_descriptors, reqWriteIdList);
- partInfo.insert_file_descriptors = new ArrayList<>();
- partInfo.delete_file_descriptors = new ArrayList<>();
- }
- hits.inc();
- } catch (CatalogException ex) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not use cached file descriptors of partition {} of table"
- + " {} for writeIdList {}", part.getPartitionName(), getFullName(),
- reqWriteIdList, ex);
- }
- misses.inc();
- missingPartitionInfos.put(part, partInfo);
- }
- }
-
- if (req.table_info_selector.want_partition_stats) {
- partInfo.setPartition_stats(part.getPartitionStatsCompressed());
+ Pair<TPartialPartitionInfo, Integer> partInfoStatus =
+ part.getPartialPartitionInfo(req, reqWriteIdList);
+ if (partInfoStatus.second != null) {
+ hits.inc();
+ numFilesFiltered += partInfoStatus.second;
+ } else {
+ misses.inc();
+ missingPartitionInfos.put(part, partInfoStatus.first);
}
-
- partInfo.setIs_marked_cached(part.isMarkedCached());
- resp.table_info.partitions.add(partInfo);
+ resp.table_info.partitions.add(partInfoStatus.first);
}
}
// In most of the cases, the prefix map only contains one item for the table location.
@@ -2373,16 +2338,6 @@ public class HdfsTable extends Table implements FeFsTable {
return resp;
}
- private int addFilteredFds(List<FileDescriptor> fds, List<THdfsFileDesc> thriftFds,
- ValidWriteIdList writeIdList) throws CatalogException {
- List<FileDescriptor> filteredFds = new ArrayList<>(fds);
- int numFilesFiltered = AcidUtils.filterFdsForAcidState(filteredFds, writeIdList);
- for (FileDescriptor fd: filteredFds) {
- thriftFds.add(fd.toThrift());
- }
- return numFilesFiltered;
- }
-
private double getFileMetadataCacheHitRate() {
long hits = metrics_.getCounter(FILEMETADATA_CACHE_HIT_METRIC).getCount();
long misses = metrics_.getCounter(FILEMETADATA_CACHE_MISS_METRIC).getCount();
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
index 38d72914c..09d6e8d12 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergTable.java
@@ -23,6 +23,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -30,6 +31,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
@@ -42,6 +44,7 @@ import org.apache.impala.analysis.IcebergPartitionTransform;
import org.apache.impala.catalog.iceberg.GroupedContentFiles;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.service.BackendConfig;
+import org.apache.impala.thrift.CatalogLookupStatus;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TCompressionCodec;
import org.apache.impala.thrift.TGetPartialCatalogObjectRequest;
@@ -55,6 +58,7 @@ import org.apache.impala.thrift.TIcebergPartitionSpec;
import org.apache.impala.thrift.TIcebergPartitionStats;
import org.apache.impala.thrift.TIcebergTable;
import org.apache.impala.thrift.TPartialPartitionInfo;
+import org.apache.impala.thrift.TSqlConstraints;
import org.apache.impala.thrift.TTable;
import org.apache.impala.thrift.TTableDescriptor;
import org.apache.impala.thrift.TTableType;
@@ -553,12 +557,6 @@ public class IcebergTable extends Table implements FeIcebergTable {
if (!col.getStats().hasNumDistinctValues()
|| snapshot.timestampMillis() >= hmsStatsTimestampMs) {
col.getStats().setNumDistinctValues(ndv);
-
- // In local catalog mode, the stats sent from the catalog are those of
- // 'hdfsTable_', not those of this class.
- Column hdfsTableCol = hdfsTable_.getColumn(col.getName());
- Preconditions.checkNotNull(hdfsTableCol);
- hdfsTableCol.getStats().setNumDistinctValues(ndv);
}
}
}
@@ -711,7 +709,7 @@ public class IcebergTable extends Table implements FeIcebergTable {
List<TIcebergPartitionSpec> params) {
List<IcebergPartitionSpec> ret = new ArrayList<>();
for (TIcebergPartitionSpec param : params) {
- // Non-partition iceberg table only has one PartitionSpec with an empty
+ // Non-partitioned iceberg table only has one PartitionSpec with an empty
// PartitionField set and a partition id
if (param.getPartition_fields() != null) {
List<IcebergPartitionField> fields = new ArrayList<>();
@@ -760,16 +758,61 @@ public class IcebergTable extends Table implements FeIcebergTable {
public TGetPartialCatalogObjectResponse getPartialInfo(
TGetPartialCatalogObjectRequest req) throws CatalogException {
Preconditions.checkState(isLoaded(), "unloaded table: %s", getFullName());
- Map<HdfsPartition, TPartialPartitionInfo> missingPartialInfos = new HashMap<>();
- TGetPartialCatalogObjectResponse resp =
- getHdfsTable().getPartialInfo(req, missingPartialInfos);
- if (resp.table_info != null) {
- // Clear HdfsTable virtual columns and add IcebergTable virtual columns.
- resp.table_info.unsetVirtual_columns();
- for (VirtualColumn vCol : getVirtualColumns()) {
- resp.table_info.addToVirtual_columns(vCol.toThrift());
+ TGetPartialCatalogObjectResponse resp = super.getPartialInfo(req);
+ Preconditions.checkState(resp.table_info != null);
+ boolean wantPartitionInfo = req.table_info_selector.want_partition_files
+ || req.table_info_selector.want_partition_metadata
+ || req.table_info_selector.want_partition_names
+ || req.table_info_selector.want_partition_stats;
+ Preconditions.checkState(!req.table_info_selector.want_hms_partition);
+ Collection<Long> partIds = req.table_info_selector.partition_ids;
+
+ if (partIds != null && partIds.isEmpty()) {
+ resp.table_info.partitions = Lists.newArrayListWithCapacity(0);
+ } else if (wantPartitionInfo || partIds != null) {
+ // Caller specified at least one piece of partition info. If they didn't explicitly
+ // specify the partitions, it means that they want the info for all partitions.
+ // (Iceberg tables are handled as unpartitioned tables, having only 1 partition.)
+ Preconditions.checkState(partIds == null || partIds.size() == 1);
+ long partId = getPartitionMap().keySet().iterator().next();
+ FeFsPartition part = (FeFsPartition) getPartitionMap().get(partId);
+ if (part == null) {
+ LOG.warn(String.format("Missing partition ID: %s, Table: %s", partId,
+ getFullName()));
+ return new TGetPartialCatalogObjectResponse().setLookup_status(
+ CatalogLookupStatus.PARTITION_NOT_FOUND);
}
+ TPartialPartitionInfo partInfo = part.getDefaultPartialPartitionInfo(req);
+ resp.table_info.partitions = Lists.newArrayList(partInfo);
+ }
+
+ // In most of the cases, the prefix map only contains one item for the table location.
+ // Here we always send it since it's small.
+ resp.table_info.setPartition_prefixes(
+ hdfsTable_.partitionLocationCompressor_.getPrefixes());
+
+ if (req.table_info_selector.want_partition_files) {
+ // TODO(todd) we are sending the whole host index even if we returned only
+ // one file -- maybe not so efficient, but the alternative is to do a bunch
+ // of cloning of file descriptors which might increase memory pressure.
+ resp.table_info.setNetwork_addresses(getHostIndex().getList());
}
+
+ if (req.table_info_selector.want_table_constraints) {
+ TSqlConstraints sqlConstraints =
+ new TSqlConstraints(getSqlConstraints().getPrimaryKeys(),
+ getSqlConstraints().getForeignKeys());
+ resp.table_info.setSql_constraints(sqlConstraints);
+ }
+ // Publish the isMarkedCached_ marker so coordinators don't need to validate
+ // it again which requires additional HDFS RPCs.
+ resp.table_info.setIs_marked_cached(isMarkedCached());
+
+ // Add IcebergTable virtual columns.
+ for (VirtualColumn vCol : getVirtualColumns()) {
+ resp.table_info.addToVirtual_columns(vCol.toThrift());
+ }
+
if (req.table_info_selector.want_iceberg_table) {
resp.table_info.setIceberg_table(Utils.getTIcebergTable(this));
if (!resp.table_info.isSetNetwork_addresses()) {
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
index 4f5521ddf..bd1113368 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
@@ -297,6 +297,11 @@ public class LocalFsTable extends LocalTable implements FeFsTable {
return null;
}
+ @Override
+ public boolean isHiveAcid() {
+ return AcidUtils.isTransactionalTable(getMetaStoreTable().getParameters());
+ }
+
@Override
public TResultSet getTableStats() {
return HdfsTable.getTableStats(this);
(impala) 02/03: IMPALA-13826: Migrate from imp to importlib in the config generator
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
laszlog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 2cf57778922aa0456eba077097be5d8ba8176e9d
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Wed Feb 5 20:06:16 2025 +0100
IMPALA-13826: Migrate from imp to importlib in the config generator
Python has deprecated the 'imp' package in Python 3.4, and removed it in
Python 3.12. The deprecation has also started throwing warnings in
versions before 3.12.
The template generator used a single call to imp.load_source to load the
template Python file. This is now replaced with code snippet published
in Python's official documentation.
Change-Id: I472d093eeaac97a380d444a1756b54f825b2d031
Reviewed-on: http://gerrit.cloudera.org:8080/22582
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Tested-by: Laszlo Gaal <la...@cloudera.com>
---
bin/generate_xml_config.py | 30 ++++++++++++++++++++++++++++--
1 file changed, 28 insertions(+), 2 deletions(-)
diff --git a/bin/generate_xml_config.py b/bin/generate_xml_config.py
index d4e7e6561..dc6e5eca8 100755
--- a/bin/generate_xml_config.py
+++ b/bin/generate_xml_config.py
@@ -40,7 +40,6 @@ REPL:
"""
from __future__ import absolute_import, division, print_function
-import imp
import os
import re
import sys
@@ -95,6 +94,33 @@ def dump_config(d, source_path, out):
print("</configuration>", file=out)
+def load_source_with_importlib(modname, filename):
+ """"Emulate imp.load_source() of Python2 for Python3 using importlib
+ Code taken from published Python documentation, see
+ https://docs.python.org/3/whatsnew/3.12.html#imp"""
+ import importlib.util
+ import importlib.machinery
+
+ loader = importlib.machinery.SourceFileLoader(modname, filename)
+ spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
+ module = importlib.util.module_from_spec(spec)
+ # The module is always executed and not cached in sys.modules.
+ # Uncomment the following line to cache the module.
+ # sys.modules[module.__name__] = module
+ loader.exec_module(module)
+ return module
+
+
+def import_template(name, module_path):
+ """Handle module import differences between Python2 and Python3"""
+ mod = None
+ if sys.version_info.major < 3:
+ import imp
+ mod = imp.load_source('template', module_path)
+ else:
+ mod = load_source_with_importlib(name, module_path)
+ return mod
+
def main():
if len(sys.argv) != 3:
print("usage: {prog} <template> <out>".format(prog=sys.argv[0]), file=sys.stderr)
@@ -102,7 +128,7 @@ def main():
_, in_path, out_path = sys.argv
try:
- mod = imp.load_source('template', in_path)
+ mod = import_template('template', in_path)
except: # noqa
print("Unable to load template: %s" % in_path, file=sys.stderr)
raise
(impala) 03/03: IMPALA-13825: Extend Docker container build to custom base images
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
laszlog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e6078b42819b3642d0030992f77ff030abf2db9f
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Wed Sep 11 19:22:23 2024 +0200
IMPALA-13825: Extend Docker container build to custom base images
Downstream system vendors, users and customers have lately expressed
interest in consuming Impala in containerized forms, taking advantage of
various specialized, hardened container base image offerings, like
container offerings based on the Wolfi project by Chainguard;
see: https://github.com/wolfi-dev.
This patch enables Impala container images to be built on top of custom
base images, and adds an implementation example that uses the publicly
available Wolfi base image.
Building a customized Docker image follows a hybrid approach. Instead of
replicating the complete Impala build process inside a Wolfi container
for a fully native binary build, it relies on an existing build platform
that is compatible with the binary packages available inside the custom
container image. For Wolfi the Impala binaries are supplied by the
Red Hat 9 build of Impala. This is made possible by the fact that major
library dependencies of Impala have the same versions on Wolfi OS and
Red Hat 9, so binaries built on Red Hat 9 can be run on Wolfi
with no changes.
The binaries produced by the regular build process are then installed
into a Docker image built on top of an explicitly specified custom base
image. The selection of a custom base image is controlled by two
environment variables:
- USE_CUSTOM_IMPALA_BASE_IMAGE (boolean):
If set to 'true', triggers the use of the custom image.
When set to 'false' or left unspecified, the Docker base image is
selected by the existing logic of matching the build platform's
operating system.
- IMPALA_CUSTOM_DOCKER_BASE (string): specifies the URI of the base image
These environment variables can be overridden from the environment,
from impala-config-branch.sh, or impala-config-local.sh.
They are reported at the end of bin/impala-config.sh where important
environment variables are listed. They are also added to the list of
variables in bin/jenkins/dockerized-impala-preserve-vars.py to ensure
that they can be used in the context of Jenkins jobs as well.
The unified script that installs Impala's required dependencies into the
container image is extended for Wolfi to handle APK packages.
A new script is added to install Bash in the Docker image if it is
missing. Impala build scripts (including the scripts used during Docker
image builds) as well as container startup scripts require Bash,
but minimal container base images usually omit it, favoring a smaller
alternative.
To improve the debugging experience for a containerized Impala
minicluster, the minicluster starter script bin/start-impala-cluster.py
is extended with the following features:
- synchronizes every launched container's timezone to the host.
This is needed for Iceberg time-travel test, which create timestamped
Iceberg metadata items in the impalad context inside a container, but
check creation/modification times of the same items in the test scripts
running on the host, outside the containers. The tests scripts have
the implicit expectation that the same local time is shared across
all these contexts, but this is not necessarily true if the host,
where tests are running is set to a timezone other than UTC.
Time sycnhronization is achieved by injecting the TZ environment
variable into the container, holding the name of the timezone used
on the host. The timezone name is taken either from the host's TZ
variable (if set), or from the host's /etc/localtime symlink,
checking the name of the timezone file it points to.
In case /etc/localtime is not a symlink (and TZ is not set on the
host), the host's /etc/localtime file is mounted into the container.
- sets up a directory for each container to collect the Java VMs error
files (hs_err_pidNNNN.log) from the containers.
- adds the --mount_sources command line parameter, which mounts the
complete $IMPALA_HOME subtree into the container at
/opt/impala/sources to make source code available inside the container
for easier debugging.
Tested by running core-mode tests in the following environments:
- Regular run (impalad running natively on the platform) on Ubuntu 20.04
- Regular run on Rocky Linux 9.2
- Dockerised run (impalad instances running in their individual
containers) using Ubuntu 20.04 containers
- Dockerised run (impalad instances running in their individual
containers) using Rocky Linux 9.2 containers
- Dockerised run (impalad instances running in their individual
containers) using Wolfi's wolfi-base containers
Change-Id: Ia5e39f399664fe66f3774caa316ed5d4df24befc
Reviewed-on: http://gerrit.cloudera.org:8080/22583
Reviewed-by: Laszlo Gaal <la...@cloudera.com>
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Reviewed-by: Jason Fehr <jf...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/impala-config.sh | 11 ++++
.../dockerized-impala-bootstrap-and-test.sh | 3 +-
bin/start-impala-cluster.py | 60 ++++++++++++++++++
docker/CMakeLists.txt | 69 ++++++++++++--------
docker/daemon_entrypoint.sh | 22 ++++++-
docker/docker-build.sh | 1 +
docker/impala_base/Dockerfile | 10 ++-
docker/impala_profile_tool/Dockerfile | 13 +++-
docker/install_bash_if_needed.sh | 74 ++++++++++++++++++++++
docker/install_os_packages.sh | 65 ++++++++++++++++++-
docker/setup_build_context.py | 5 ++
tests/common/impala_connection.py | 4 +-
12 files changed, 298 insertions(+), 39 deletions(-)
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 689a3a069..b86c46393 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -297,6 +297,15 @@ export IMPALA_DATASKETCHES_VERSION=6.0.0
export IMPALA_REDHAT7_DOCKER_BASE=${IMPALA_REDHAT7_DOCKER_BASE:-"centos:centos7.9.2009"}
export IMPALA_REDHAT8_DOCKER_BASE=${IMPALA_REDHAT8_DOCKER_BASE:-"rockylinux:8.5"}
export IMPALA_REDHAT9_DOCKER_BASE=${IMPALA_REDHAT9_DOCKER_BASE:-"rockylinux:9.2"}
+# Some users may want to use special, hardened base images for increased security.
+# These images are usually not related to the OS where the build is running.
+# The following environment variables allow a specific base image to be specified
+# directly, without relying on the implicit build platform identification in
+# CMakeLists.txt.
+# Images published by Chainguard and the Wolfi project are known to be used, so the
+# publicly available Wolfi base image is used as a default example.
+export IMPALA_CUSTOM_DOCKER_BASE=${IMPALA_CUSTOM_DOCKER_BASE:-"cgr.dev/chainguard/wolfi-base:latest"}
+export USE_CUSTOM_IMPALA_BASE_IMAGE=${USE_CUSTOM_IMPALA_BASE_IMAGE:-false}
# Selects the version of Java to use when start-impala-cluster.py starts with container
# images (created via e.g. 'make docker_debug_java11_images'). The Java version used in
@@ -1230,6 +1239,8 @@ echo "IMPALA_SYSTEM_PYTHON2 = $IMPALA_SYSTEM_PYTHON2"
echo "IMPALA_SYSTEM_PYTHON3 = $IMPALA_SYSTEM_PYTHON3"
echo "IMPALA_BUILD_THREADS = $IMPALA_BUILD_THREADS"
echo "NUM_CONCURRENT_TESTS = $NUM_CONCURRENT_TESTS"
+echo "USE_CUSTOM_IMPALA_BASE_IMAGE = $USE_CUSTOM_IMPALA_BASE_IMAGE"
+echo "IMPALA_CUSTOM_DOCKER_BASE = $IMPALA_CUSTOM_DOCKER_BASE"
# Kerberos things. If the cluster exists and is kerberized, source
# the required environment. This is required for any hadoop tool to
diff --git a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
index 07ab2612d..d2493de49 100755
--- a/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
+++ b/bin/jenkins/dockerized-impala-bootstrap-and-test.sh
@@ -36,7 +36,8 @@ source ./bin/bootstrap_system.sh
# to preserve additional variables.
./bin/jenkins/dockerized-impala-preserve-vars.py \
EE_TEST EE_TEST_FILES JDBC_TEST EXPLORATION_STRATEGY CMAKE_BUILD_TYPE \
- IMPALA_DOCKER_JAVA
+ IMPALA_DOCKER_JAVA IMPALA_CUSTOM_DOCKER_BASE USE_CUSTOM_IMPALA_BASE_IMAGE \
+ IMPALA_TOOLCHAIN_HOST
# Execute the tests using su to re-login so that group change made above
# setup_docker takes effect. This does a full re-login and does not stay
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 725525d5b..292604b3f 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -54,6 +54,7 @@ KUDU_MASTER_HOSTS = os.getenv("KUDU_MASTER_HOSTS", "127.0.0.1")
DEFAULT_IMPALA_MAX_LOG_FILES = os.environ.get("IMPALA_MAX_LOG_FILES", 10)
INTERNAL_LISTEN_HOST = os.getenv("INTERNAL_LISTEN_HOST", "localhost")
TARGET_FILESYSTEM = os.getenv("TARGET_FILESYSTEM") or "hdfs"
+HOST_TZ = os.getenv("TZ", None)
# Options
parser = OptionParser()
@@ -133,6 +134,9 @@ parser.add_option("--docker_auto_ports", dest="docker_auto_ports",
"(Beewax, HS2, Web UIs, etc), which avoids collisions with other "
"running processes. If false, ports are mapped to the same ports "
"on localhost as the non-docker impala cluster.")
+parser.add_option("--mount_sources", dest="mount_sources", action="store_true",
+ help="Mount the $IMPALA_HOME directory as /opt/impala/sources into "
+ "the containers for easier debugging.")
parser.add_option("--data_cache_dir", dest="data_cache_dir", default=None,
help="This specifies a base directory in which the IO data cache will "
"use.")
@@ -254,6 +258,9 @@ def build_java_tool_options(jvm_debug_port=None):
"""Construct the value of the JAVA_TOOL_OPTIONS environment variable to pass to
daemons."""
java_tool_options = ""
+ # In a Docker container the Java error file location is always fixed.
+ if options.docker_network is not None:
+ java_tool_options = "-XX:ErrorFile=/opt/impala/java-error/hs_err_pid_%p.log"
if jvm_debug_port is not None:
java_tool_options = ("-agentlib:jdwp=transport=dt_socket,address={debug_port}," +
"server=y,suspend=n ").format(debug_port=jvm_debug_port) + java_tool_options
@@ -988,6 +995,43 @@ class DockerMiniClusterOperations(object):
env_args = ["-e", "HADOOP_USER_NAME={0}".format(getpass.getuser()),
"-e", "JAVA_TOOL_OPTIONS={0}".format(
build_java_tool_options(DEFAULT_IMPALAD_JVM_DEBUG_PORT))]
+
+ # Calculate the timezone to pass into the container.
+ # Mounting /etc/localtime into the container does not work when /etc/localtime is a
+ # symbolic link to a real timezone file inside /usr/share/zoneinfo: Linux resolves
+ # the symlink before performing the bind mount, so you can't create a symlink within
+ # the container.
+ # Set the timezone by injecting the TZ environment variable with the desired timezone
+ # string instead. Initialize the env var from the host's TZ variable if it exists,
+ # or calculate the value (the timezone specifier) from the name of the timezone file
+ # pointed to by /etc/localtime, if it is a symlink.
+ # If /etc/localtime is a real file, and TZ is undefined on the host, then mount
+ # /etc/localtime into the container
+ timezone_as_env_var = True
+ timezone_as_mount = False
+
+ if HOST_TZ is None:
+ try:
+ if os.path.islink("/etc/localtime"):
+ # This is a symlink, so figure out where it points, cut the prefix, and hope
+ # we'll get a timezone spec. Don't confuse realpath() and relpath() here!
+ timezone_string = os.path.realpath("/etc/localtime")
+ timezone_string = os.path.relpath(timezone_string, "/usr/share/zoneinfo")
+ elif os.path.isfile("/etc/localtime"):
+ # This is a real file, and we'll just have to mount it into the container
+ timezone_as_env_var = False
+ timezone_as_mount = True
+ else:
+ timezone_as_env_var = False
+ timezone_as_mount = False
+ LOG.warning("Unable to determine local timezone, "
+ "containers will user their default timezones.")
+ except OSError as ex:
+ timezone_as_env_var = False
+ LOG.error("Unable to map /etc/localtime to a timezone name. Reported error"
+ "is {0}".format(ex))
+ if timezone_as_env_var:
+ env_args += ["-e", "TZ={0}".format(timezone_string)]
# The container build processes tags the generated image with the daemon name.
debug_build = options.build_type == "debug" or (options.build_type == "latest" and
os.path.basename(os.path.dirname(os.readlink("be/build/latest"))) == "debug")
@@ -1009,6 +1053,22 @@ class DockerMiniClusterOperations(object):
if not os.path.isdir(log_dir):
os.makedirs(log_dir)
mount_args += ["--mount", "type=bind,src={0},dst=/opt/impala/logs".format(log_dir)]
+ # Collect Java error files hs_err_pidNNN.log in a unique subdirectory per daemon to
+ # avoid any potential interaction between containers, which should be isolated.
+ java_error_dir = os.path.join(IMPALA_HOME, options.log_dir, host_name, "java-error")
+ if not os.path.isdir(java_error_dir):
+ os.makedirs(java_error_dir)
+ mount_args += ["--mount", "type=bind,src={0},dst=/opt/impala/java-errors".format(
+ java_error_dir)]
+ # If /etc/localtime was found to be a real file instead of a symlink, then mount it
+ # into the container to ensure consistent clocks between the host and the Impala
+ # containers. This is important for logs as well as Iceberg tests.
+ if timezone_as_mount:
+ mount_args += ["--mount",
+ "type=bind,src=/etc/localtime,dst=/etc/localtime,readonly"]
+ if options.mount_sources:
+ mount_args += ["--mount",
+ "type=bind,src={0},dst=/opt/impala/sources,readonly".format(IMPALA_HOME)]
# Add entries to the container's /etc/hosts file for the Docker host and the
# gateway from the container to the host. These are needed for stable reverse
# name resolution of the host's and the gateway's IP addresses
diff --git a/docker/CMakeLists.txt b/docker/CMakeLists.txt
index 4cf61c33b..e9c232eb3 100644
--- a/docker/CMakeLists.txt
+++ b/docker/CMakeLists.txt
@@ -32,40 +32,53 @@ cmake_host_system_information(RESULT OS_DISTRIB_VERSION_ID QUERY DISTRIB_VERSION
set(QUICKSTART_BASE_IMAGE "UNSUPPORTED")
set(DISTRO_BASE_IMAGE "UNSUPPORTED")
+MESSAGE(STATUS "OS_DISTRIB_ID is: ${OS_DISTRIB_ID}")
+MESSAGE(STATUS "OS_DISTRIB_VERSION_ID is: ${OS_DISTRIB_VERSION_ID}")
+
# The CMake variables are using information from /etc/os-release.
# A database of /etc/os-release files is available at
# https://github.com/chef/os_release
# These comparisons are based on those values.
-if(${OS_DISTRIB_ID} STREQUAL "ubuntu")
- if(${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
- ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04" OR
- ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
- ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
- set(DISTRO_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
- set(QUICKSTART_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
- endif()
- if (${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
- ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04")
- set(PIP "python-pip")
- elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
- ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
- set(PIP "python3-pip")
- endif()
-elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR
- ${OS_DISTRIB_ID} STREQUAL "rocky" OR
- ${OS_DISTRIB_ID} STREQUAL "almalinux" OR
- ${OS_DISTRIB_ID} STREQUAL "centos")
- # The Quickstart images currently don't support using a Redhat
- # base image, so this doesn't set QUICKSTART_BASE_IMAGE.
- if(${OS_DISTRIB_VERSION_ID} MATCHES "7.*")
- set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}")
- elseif(${OS_DISTRIB_VERSION_ID} MATCHES "8.*")
- set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}")
- elseif(${OS_DISTRIB_VERSION_ID} MATCHES "9.*")
- set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT9_DOCKER_BASE}")
+# A custom base image can be specified directly in the IMPALA_CUSTOM_DOCKER_BASE
+# environment variable if USE_CUSTOM_IMPALA_BASE_IMAGE to set to true in the environment.
+MESSAGE(STATUS "USE_CUSTOM_IMPALA_BASE_IMAGE is: $ENV{USE_CUSTOM_IMPALA_BASE_IMAGE}")
+if( "$ENV{USE_CUSTOM_IMPALA_BASE_IMAGE}" STREQUAL "true" )
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_CUSTOM_DOCKER_BASE}")
+ # Don't publish a QuickStart image on a custom base image: it is unknown if it has
+ # all prerequisites, so don't set QUICKSTART_BASE_IMAGE.
+ MESSAGE(STATUS "Picked custom base image: ${DISTRO_BASE_IMAGE}")
+else()
+ if(${OS_DISTRIB_ID} STREQUAL "ubuntu")
+ if(${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
+ set(DISTRO_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
+ set(QUICKSTART_BASE_IMAGE "ubuntu:${OS_DISTRIB_VERSION_ID}")
+ endif()
+ if (${OS_DISTRIB_VERSION_ID} STREQUAL "16.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "18.04")
+ set(PIP "python-pip")
+ elseif (${OS_DISTRIB_VERSION_ID} STREQUAL "20.04" OR
+ ${OS_DISTRIB_VERSION_ID} STREQUAL "22.04")
+ set(PIP "python3-pip")
+ endif()
+ elseif(${OS_DISTRIB_ID} STREQUAL "rhel" OR
+ ${OS_DISTRIB_ID} STREQUAL "rocky" OR
+ ${OS_DISTRIB_ID} STREQUAL "almalinux" OR
+ ${OS_DISTRIB_ID} STREQUAL "centos")
+ # The Quickstart images currently don't support using a Redhat
+ # base image, so this doesn't set QUICKSTART_BASE_IMAGE.
+ if(${OS_DISTRIB_VERSION_ID} MATCHES "7.*")
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT7_DOCKER_BASE}")
+ elseif(${OS_DISTRIB_VERSION_ID} MATCHES "8.*")
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT8_DOCKER_BASE}")
+ elseif(${OS_DISTRIB_VERSION_ID} MATCHES "9.*")
+ set(DISTRO_BASE_IMAGE "$ENV{IMPALA_REDHAT9_DOCKER_BASE}")
+ endif()
endif()
+ MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}")
endif()
-MESSAGE(STATUS "Picked docker base image based on host OS: ${DISTRO_BASE_IMAGE}")
if (NOT ${DISTRO_BASE_IMAGE} STREQUAL "UNSUPPORTED")
diff --git a/docker/daemon_entrypoint.sh b/docker/daemon_entrypoint.sh
index 643d680c5..897987816 100755
--- a/docker/daemon_entrypoint.sh
+++ b/docker/daemon_entrypoint.sh
@@ -36,17 +36,24 @@ DISTRIBUTION=Unknown
if [[ -f /etc/redhat-release ]]; then
echo "Identified Redhat image."
DISTRIBUTION=Redhat
-else
+elif [[ -f /etc/lsb-release ]]; then
source /etc/lsb-release
if [[ $DISTRIB_ID == Ubuntu ]]; then
echo "Identified Ubuntu image."
DISTRIBUTION=Ubuntu
fi
+# Check /etc/os-release last: it exists on Red Hat and Ubuntu systems as well
+elif [[ -f /etc/os-release ]]; then
+ source /etc/os-release
+ if [[ $ID == wolfi || $ID == chainguard ]]; then
+ echo "Identified Wolfi-based system."
+ DISTRIBUTION=Chainguard
+ fi
fi
if [[ $DISTRIBUTION == Unknown ]]; then
echo "ERROR: Did not detect supported distribution."
- echo "Only Ubuntu and Redhat-based distributions are supported."
+ echo "Only Ubuntu, Red Hat (or related), or Wolfi distributions are supported."
exit 1
fi
@@ -78,6 +85,17 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
echo "Detected Java 8"
JAVA_HOME=/usr/lib/jvm/jre-1.8.0
fi
+elif [[ $DISTRIBUTION == Chainguard ]]; then
+ if [[ -d /usr/lib/jvm/java-17-openjdk ]] ; then
+ echo "Detected Java 17"
+ JAVA_HOME=/usr/lib/jvm/java-17-openjdk
+ elif [[ -d /usr/lib/jvm/java-11-openjdk ]] ; then
+ echo "Detected Java 11"
+ JAVA_HOME=/usr/lib/jvm/java-11-openjdk
+ elif [[ -d /usr/lib/jvm/java-1.8-openjdk ]] ; then
+ echo "Detected Java 8"
+ JAVA_HOME=/usr/lib/jvm/java-1.8-openjdk
+ fi
fi
if [[ $JAVA_HOME == Unknown ]]; then
diff --git a/docker/docker-build.sh b/docker/docker-build.sh
index e05eb605e..d881e441f 100755
--- a/docker/docker-build.sh
+++ b/docker/docker-build.sh
@@ -32,6 +32,7 @@ ARGS+=("--build-arg" 'VCS_TYPE=git')
ARGS+=("--build-arg" 'VCS_URL=https://gitbox.apache.org/repos/asf/impala.git')
ARGS+=("--build-arg" "VERSION=$VERSION")
ARGS+=("--build-arg" "VCS_REF=$VCS_REF")
+ARGS+=("--progress=plain")
# Add caller-provided arguments to end.
ARGS+=("$@")
diff --git a/docker/impala_base/Dockerfile b/docker/impala_base/Dockerfile
index 2d84d2514..463887a1c 100644
--- a/docker/impala_base/Dockerfile
+++ b/docker/impala_base/Dockerfile
@@ -22,8 +22,16 @@ FROM ${BASE_IMAGE}
# The level of debugging tools to install is set by the argument to "--install-debug-tools".
ARG INSTALL_OS_PACKAGES_ARGS="--install-debug-tools none"
+# Switch to root. Needed by hardened base images that don't default to the root user.
+USER root
+# Install Bash, if missing from the base image: not all of them have it by default,
+# but Impala scripts require it.
+ADD --chown=root:root --chmod=755 helper/install_bash_if_needed.sh /root
+RUN /root/install_bash_if_needed.sh
+
# Install minimal dependencies required for Impala services to run.
-ADD helper/install_os_packages.sh /root
+ADD --chown=root:root --chmod=755 helper/install_os_packages.sh /root
+
RUN /root/install_os_packages.sh ${INSTALL_OS_PACKAGES_ARGS}
# Use a non-privileged impala user to run the daemons in the container.
diff --git a/docker/impala_profile_tool/Dockerfile b/docker/impala_profile_tool/Dockerfile
index 78fcae827..cd0e15ce0 100644
--- a/docker/impala_profile_tool/Dockerfile
+++ b/docker/impala_profile_tool/Dockerfile
@@ -22,11 +22,19 @@ FROM ${BASE_IMAGE}
# If set to "--install-debug-tools full", then extra utilities will be installed.
ARG INSTALL_OS_PACKAGES_ARGS="--install-debug-tools full"
+# Switch to root. Needed by hardened base images that don't default to the root user.
+USER root
+# Install Bash, if missing from the base image: not all of them have it by default,
+# but Impala scripts require it.
+ADD --chown=root:root --chmod=755 helper/install_bash_if_needed.sh /root
+RUN /root/install_bash_if_needed.sh
+
# Install dependencies required for Impala utility binaries to run, plus
# some useful utilities.
# TODO: ideally we wouldn't depend on the JVM libraries, but currently the JNI code
# in be/ is not cleanly separated from the code that doesn't use JNI.
-ADD helper/install_os_packages.sh /root
+ADD --chown=root:root --chmod=755 helper/install_os_packages.sh /root
+
RUN /root/install_os_packages.sh ${INSTALL_OS_PACKAGES_ARGS}
# Use a non-privileged impala user to run the processes in the container.
@@ -43,7 +51,8 @@ COPY --chown=impala lib /opt/impala/lib
WORKDIR /opt/impala/
-ENTRYPOINT ["/opt/impala/bin/utility_entrypoint.sh", "/opt/impala/bin/impala-profile-tool",\
+ENTRYPOINT ["/opt/impala/bin/utility_entrypoint.sh", \
+ "/opt/impala/bin/impala-profile-tool",\
"-logtostderr"]
LABEL name="Apache Impala Profile Tool" \
diff --git a/docker/install_bash_if_needed.sh b/docker/install_bash_if_needed.sh
new file mode 100755
index 000000000..be0b933cc
--- /dev/null
+++ b/docker/install_bash_if_needed.sh
@@ -0,0 +1,74 @@
+#!/bin/sh
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# This installs bash if the base Docker image does not have it.
+# Bash is needed for the other helper shell scripts for Impala images,
+# so the first step is to make it available.
+
+set -eu
+
+# Default level of extra debugging tools, controlled by the --install-debug-tools flag.
+INSTALL_DEBUG_TOOLS=none
+
+DRY_RUN=false
+
+if command -v bash; then
+ echo "Bash found, skipping installation."
+ exit 0
+fi
+
+# This can get more detailed if there are specific steps
+# for specific versions, but at the moment the distribution
+# is all we need.
+DISTRIBUTION=Unknown
+if [ -f /etc/redhat-release ]; then
+ echo "Identified Redhat system."
+ DISTRIBUTION=Redhat
+elif [ -f /etc/lsb-release ]; then
+ . /etc/lsb-release
+ if [ $DISTRIB_ID = Ubuntu ]; then
+ echo "Identified Ubuntu system."
+ DISTRIBUTION=Ubuntu
+ fi
+# Check /etc/os-release last: it exists on Red Hat and Ubuntu systems as well
+elif [ -f /etc/os-release ]; then
+ source /etc/os-release
+ if [ $ID = wolfi -o $ID = chainguard ]; then
+ echo "Identified Wolfi-based system."
+ DISTRIBUTION=Chainguard
+ fi
+fi
+
+if [ $DISTRIBUTION = Unknown ]; then
+ echo "ERROR: Did not detect supported distribution."
+ echo "Only Ubuntu, Red Hat (or related), or Wolfi base images are supported."
+ exit 1
+fi
+
+# Install minimal set of files.
+# Optionally install extra debug tools.
+if [ $DISTRIBUTION = Ubuntu ]; then
+ export DEBIAN_FRONTEND=noninteractive
+ apt-get update
+ apt-get install -y bash
+elif [ $DISTRIBUTION = Redhat ]; then
+ yum install -y bash
+elif [ $DISTRIBUTION = Chainguard ]; then
+ apk add --no-cache --no-interactive bash
+fi
+
diff --git a/docker/install_os_packages.sh b/docker/install_os_packages.sh
index 4422fd579..d387e5097 100755
--- a/docker/install_os_packages.sh
+++ b/docker/install_os_packages.sh
@@ -29,7 +29,7 @@ INSTALL_DEBUG_TOOLS=none
JAVA_VERSION=8
DRY_RUN=false
PKG_LIST=""
-NON_PKG_NAMES=(apt-get yum install update)
+NON_PKG_NAMES=(apt-get yum apk install update add)
function print_usage {
echo "install_os_packages.sh - Helper script to install OS dependencies"
@@ -105,7 +105,7 @@ DISTRIBUTION=Unknown
if [[ -f /etc/redhat-release ]]; then
echo "Identified Redhat system."
DISTRIBUTION=Redhat
-else
+elif [[ -f /etc/lsb-release ]]; then
source /etc/lsb-release
if [[ $DISTRIB_ID == Ubuntu ]]; then
echo "Identified Ubuntu system."
@@ -118,11 +118,18 @@ else
exit 1
fi
fi
+# Check /etc/os-release last: it exists on Red Hat and Ubuntu systems as well
+elif [[ -f /etc/os-release ]]; then
+ source /etc/os-release
+ if [[ $ID == wolfi || $ID == chainguard ]]; then
+ echo "Identified Wolfi-based system."
+ DISTRIBUTION=Chainguard
+ fi
fi
if [[ $DISTRIBUTION == Unknown ]]; then
echo "ERROR: Did not detect supported distribution."
- echo "Only Ubuntu and Redhat-based distributions are supported."
+ echo "Only Ubuntu, Red Hat (or related), or Wolfi base images are supported."
exit 1
fi
@@ -202,6 +209,49 @@ elif [[ $DISTRIBUTION == Redhat ]]; then
vim \
which
fi
+elif [[ $DISTRIBUTION == Chainguard ]]; then
+ # Package inventory:
+ # glibc-locale-en and posix-libc-utils: for locale and 'locale' tool support
+ # shadow: for groupadd and friends
+ wrap apk add --no-cache --no-interactive \
+ glibc-locale-posix \
+ glibc-locale-en \
+ glibc-iconv \
+ posix-libc-utils \
+ localedef \
+ cyrus-sasl \
+ krb5-libs \
+ openssl \
+ openldap-dev \
+ openjdk-${JAVA_VERSION}-jre \
+ openjdk-${JAVA_VERSION}-default-jvm \
+ shadow \
+ tzdata
+
+ # Set up /etc/localtime so that the container has a default local timezone.
+ # Make this UTC.
+ ln -sf /usr/share/zoneinfo/UTC /etc/localtime
+
+ if [[ $INSTALL_DEBUG_TOOLS == basic || $INSTALL_DEBUG_TOOLS == full ]]; then
+ echo "Installing basic debug tools"
+ wrap apk add --no-cache --no-interactive \
+ gdb \
+ openjdk-${JAVA_VERSION}-default-jdk
+ fi
+
+ if [[ $INSTALL_DEBUG_TOOLS == full ]]; then
+ echo "Installing full debug tools"
+ wrap apk add --no-cache --no-interactive \
+ bind-tools \
+ curl \
+ iproute2 \
+ iputils \
+ less \
+ nmap \
+ sudo \
+ tzutils \
+ vim
+ fi
fi
if $DRY_RUN; then
@@ -227,9 +277,16 @@ if ! command -v pgrep ; then
exit 1
fi
+# Java must be accessible for both the frontend and the backend. Verify it is present.
+if ! command -v java; then
+ echo "ERROR: Java cannot be found."
+ exit 1
+fi
+
# Impala will fail to start if the permissions on /var/tmp are not set to include
# the sticky bit (i.e. +t). Some versions of Redhat UBI images do not have
# this set by default, so specifically set the sticky bit for both /tmp and /var/tmp.
+mkdir -p /var/tmp
chmod a=rwx,o+t /var/tmp /tmp
# To minimize the size for the Docker image, clean up any unnecessary files.
@@ -239,4 +296,6 @@ if [[ $DISTRIBUTION == Ubuntu ]]; then
elif [[ $DISTRIBUTION == Redhat ]]; then
yum clean all
rm -rf /var/cache/yum/*
+elif [[ $DISTRIBUTION == Chainguard ]]; then
+ rm -rf /var/cache/apk/*
fi
diff --git a/docker/setup_build_context.py b/docker/setup_build_context.py
index b46d88043..66b5e7eb6 100755
--- a/docker/setup_build_context.py
+++ b/docker/setup_build_context.py
@@ -179,6 +179,11 @@ for kudu_lib_dir in kudu_lib_dirs:
if not found_kudu_so:
raise Exception("No Kudu shared object found in search path: {0}".format(kudu_lib_dirs))
+# Add script for installing Bash: Impala scripts need it, but some minimal base images
+# may either omit shells altogether, or contain a simpler, smaller variant, e.g. sh.
+symlink_file_into_dir(
+ os.path.join(IMPALA_HOME, "docker/install_bash_if_needed.sh"), HELPER_DIR)
+
# Add script for installing OS packages
symlink_file_into_dir(
os.path.join(IMPALA_HOME, "docker/install_os_packages.sh"), HELPER_DIR)
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index 18e882c32..5e1d06266 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -131,8 +131,8 @@ def collect_default_query_options(options, name, val, kind):
return
name = name.lower()
val = str(val).strip('"')
- if ',' in val:
- # Value is a list. Wrap it with double quote.
+ if ',' in val or '/' in val:
+ # Value is a list or a timezone name containing a slash. Wrap it with double quotes.
val = '"{}"'.format(val)
if not val:
# Value is optional with None as default.