You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2026/09/23 20:19:39 UTC
(impala) branch master updated (820e108c8 -> d757d3dfe)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from 820e108c8 IMPALA-15374: Roll back current_row_ when undoing a level read ahead
new a95c64c75 IMPALA-15381: Reduce reservoir RNG state
new 8e7626ce2 IMPALA-15346: Fix IllegalStateException in createInsertEvents
new d757d3dfe IMPALA-15397: Update test after S3 reports encrypted
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exprs/aggregate-functions-ir.cc | 18 +++++++-----------
.../org/apache/impala/service/CatalogOpExecutor.java | 11 ++++++++---
.../queries/QueryTest/alloc-fail-init.test | 4 ++--
tests/custom_cluster/test_events_custom_configs.py | 12 ++++++++++++
tests/util/filesystem_utils.py | 2 +-
5 files changed, 30 insertions(+), 17 deletions(-)
(impala) 02/03: IMPALA-15346: Fix IllegalStateException in createInsertEvents
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8e7626ce231edca0807e7cf423923bed484fe4d4
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Sep 14 10:28:32 2026 +0800
IMPALA-15346: Fix IllegalStateException in createInsertEvents
This fixes an IllegalStateException in createInsertEvents when inserting
empty results into an unpartitioned table.
The exception is thrown in CatalogOpExecutor.createInsertEvents() when
asserting the number of updated partitions to be 1. The map of updated
partitions is created in CatalogOpExecutor.getFileMetadata() which loads
file metadata like checksums. It unintentionally skips adding partitions
that have an empty new file list by putting the put() statement in the
for-loop that iterates the file list. This patch fixes it by moving the
put() statement outside that for-loop. Also simplifies the case of empty
new file list.
The Preconditions check is updated to log the actual number of updated
partitions.
Testing
- Added custom-cluster test
Change-Id: I0eb366f55f4e95b229a57297e42d14760b31be21
Reviewed-on: http://gerrit.cloudera.org:8080/24835
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../java/org/apache/impala/service/CatalogOpExecutor.java | 11 ++++++++---
tests/custom_cluster/test_events_custom_configs.py | 12 ++++++++++++
2 files changed, 20 insertions(+), 3 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 2ce0f5da1..4a5299ee1 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -8316,8 +8316,14 @@ public class CatalogOpExecutor {
Map<String, List<FileMetadata>> fileMetadata = Maps.newHashMap();
int numFiles = 0;
for (Map.Entry<String, TUpdatedPartition> e : updatedPartitions.entrySet()) {
- numFiles += e.getValue().getFiles().size();
- List<FileMetadata> files = new ArrayList<>(e.getValue().getFiles().size());
+ int numPartFiles = e.getValue().getFiles().size();
+ if (numPartFiles == 0) {
+ fileMetadata.put(e.getKey(), Collections.emptyList());
+ continue;
+ }
+ numFiles += numPartFiles;
+ List<FileMetadata> files = new ArrayList<>(numPartFiles);
+ fileMetadata.put(e.getKey(), files);
for (String file : e.getValue().getFiles()) {
loaders.add(new Pair<>(file, () -> {
FileChecksum checksum = null;
@@ -8341,7 +8347,6 @@ public class CatalogOpExecutor {
return new FileMetadata(file, checksum, acidDirPath);
}));
updaters.add((result) -> files.add((FileMetadata) result));
- fileMetadata.put(e.getKey(), files);
}
}
diff --git a/tests/custom_cluster/test_events_custom_configs.py b/tests/custom_cluster/test_events_custom_configs.py
index fff17cb65..b9775b9c8 100644
--- a/tests/custom_cluster/test_events_custom_configs.py
+++ b/tests/custom_cluster/test_events_custom_configs.py
@@ -1954,6 +1954,18 @@ class TestEventProcessingCustomConfigs(TestEventProcessingCustomConfigsBase):
_check_insert_events(tbl1, 4, skip_events=1, part='partition(year=2024)')
_check_insert_events(tbl2, 4, skip_events=0)
+ @CustomClusterTestSuite.with_args(disable_log_buffering=True)
+ def test_insert_empty_result_set_nonpartitioned(self, unique_database):
+ self.execute_query("create table {}.tbl (i int)".format(unique_database))
+ # Insert empty result set to an empty table
+ self.execute_query("insert into {}.tbl select 1 from functional.alltypestiny limit 0"
+ .format(unique_database))
+ self.execute_query("insert into {}.tbl select 1".format(unique_database))
+ # Insert empty result set to a non-empty table
+ self.execute_query("insert into {}.tbl select 1 from functional.alltypestiny limit 0"
+ .format(unique_database))
+ # Asserts no errors by using expected_count=0
+ self.assert_catalogd_log_contains("INFO", "Failed to fire insert events for table", 0)
@SkipIfFS.hive
class TestEventProcessingWithImpala(TestEventProcessingCustomConfigsBase):
(impala) 03/03: IMPALA-15397: Update test after S3 reports encrypted
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit d757d3dfe5c7cdcc48aed2064ed28bbed3ef9ffe
Author: Michael Smith <mi...@cloudera.com>
AuthorDate: Tue Sep 22 09:58:46 2026 -0700
IMPALA-15397: Update test after S3 reports encrypted
After IMPALA-15291, S3 files now report as encrypted, so update
IS_ENCRYPTED for test_io_metrics.
Change-Id: I3f6739353d77a1fe1ce1b21cbab5009879f69c3b
Reviewed-on: http://gerrit.cloudera.org:8080/24915
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/util/filesystem_utils.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/util/filesystem_utils.py b/tests/util/filesystem_utils.py
index 5aac1b601..4cb1f1647 100644
--- a/tests/util/filesystem_utils.py
+++ b/tests/util/filesystem_utils.py
@@ -37,7 +37,7 @@ IS_OSS = FILESYSTEM == "oss"
IS_OBS = FILESYSTEM == "obs"
IS_OZONE = FILESYSTEM == "ozone"
IS_EC = os.getenv("ERASURE_CODING") == "true"
-IS_ENCRYPTED = os.getenv("USE_OZONE_ENCRYPTION") == "true"
+IS_ENCRYPTED = os.getenv("USE_OZONE_ENCRYPTION") == "true" or IS_S3
# This condition satisfies both the states where one can assume a default fs
# - The environment variable is set to an empty string.
# - Tne environment variables is unset ( None )
(impala) 01/03: IMPALA-15381: Reduce reservoir RNG state
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a95c64c756f66156399f9e4de2a5067f34765e37
Author: Aleksandr Efimov <dz...@gmail.com>
AuthorDate: Tue Sep 22 09:09:07 2026 +0300
IMPALA-15381: Reduce reservoir RNG state
boost::mt19937_64 makes ReservoirSampleState 2536 bytes. FreePool
rounds the allocation to 4 KiB, so sampling aggregates use 4624
bytes per function and group with their initial sample array.
Use pcg32 to reduce the state to 48 bytes. The sampling sequence
changes, so approximate results can change.
Testing:
- Full release build
- HistogramTest.*
- Large appx_median(), sample(), and histogram() queries
- Updated the Init() allocation size in alloc-fail-init.test
Change-Id: I9e046e3f0c1b91f1761ce578601a330bd864eb84
Assisted-by: gpt-5.6-sol (OpenAI Codex)
Reviewed-on: http://gerrit.cloudera.org:8080/24896
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exprs/aggregate-functions-ir.cc | 18 +++++++-----------
.../queries/QueryTest/alloc-fail-init.test | 4 ++--
2 files changed, 9 insertions(+), 13 deletions(-)
diff --git a/be/src/exprs/aggregate-functions-ir.cc b/be/src/exprs/aggregate-functions-ir.cc
index 48e703306..8b72f7ec1 100644
--- a/be/src/exprs/aggregate-functions-ir.cc
+++ b/be/src/exprs/aggregate-functions-ir.cc
@@ -18,13 +18,11 @@
#include "exprs/aggregate-functions.h"
#include <algorithm>
+#include <cmath>
#include <map>
+#include <random>
#include <sstream>
#include <utility>
-#include <cmath>
-
-#include <boost/random/mersenne_twister.hpp>
-#include <boost/random/uniform_int.hpp>
#include "codegen/impala-ir.h"
#include "common/logging.h"
@@ -46,14 +44,13 @@
#include "thirdparty/datasketches/theta_union.hpp"
#include "thirdparty/datasketches/theta_intersection.hpp"
#include "thirdparty/datasketches/kll_sketch.hpp"
+#include "thirdparty/pcg-cpp-0.98/include/pcg_random.hpp"
#include "util/arithmetic-util.h"
#include "util/mpfit-util.h"
#include "util/pretty-printer.h"
#include "common/names.h"
-using boost::uniform_int;
-using boost::mt19937_64;
using std::make_pair;
using std::map;
using std::min_element;
@@ -62,6 +59,7 @@ using std::pop_heap;
using std::push_heap;
using std::string;
using std::stringstream;
+using std::uniform_int_distribution;
namespace {
// Threshold for each precision where it's better to use linear counting instead
@@ -1923,10 +1921,8 @@ class ReservoirSampleState {
// Number of values over which the samples were collected.
int64_t source_size_;
- // Random number generator for generating 64-bit integers
- // Replace ranlux64_3 with mt19937_64 for better performance. See boost benchmark at
- // https://www.boost.org/doc/libs/1_74_0/doc/html/boost_random/performance.html
- mt19937_64 rng_;
+ // Random number generator used to select samples from the input stream.
+ pcg32 rng_;
// True if the array of samples is in the same memory allocation as this object. If
// false, this object is responsible for freeing the memory.
@@ -1959,7 +1955,7 @@ class ReservoirSampleState {
// Returns a random integer in the range [0, max].
int64_t GetNext64(int64_t max) {
- uniform_int<int64_t> dist(0, max);
+ uniform_int_distribution<int64_t> dist(0, max);
return dist(rng_);
}
};
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alloc-fail-init.test b/testdata/workloads/functional-query/queries/QueryTest/alloc-fail-init.test
index c6d77d6a0..5a841f1d1 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alloc-fail-init.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alloc-fail-init.test
@@ -13,7 +13,7 @@ FunctionContext::Allocate() failed to allocate 1 bytes.
---- QUERY
select sample(timestamp_col) from functional.alltypes
---- CATCH
-FunctionContext::Allocate() failed to allocate 2536 bytes.
+FunctionContext::Allocate() failed to allocate 48 bytes.
====
---- QUERY
select group_concat(string_col) from functional.alltypes
@@ -58,7 +58,7 @@ FunctionContextImpl::AllocateForResults() failed to allocate 120 bytes.
---- QUERY
select appx_median(int_col) from functional.alltypes
---- CATCH
-FunctionContext::Allocate() failed to allocate 2536 bytes.
+FunctionContext::Allocate() failed to allocate 48 bytes.
====
---- QUERY
select to_date(now())