You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by ki...@apache.org on 2025/11/05 09:41:36 UTC
(doris) branch auto-pick-57629-branch-3.1 updated (ade9cab1933 -> 6b2adaa590b)
This is an automated email from the ASF dual-hosted git repository.
kirs pushed a change to branch auto-pick-57629-branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
from ade9cab1933 [chore](dep)Exclude AWS SDK bundle to reduce dependency size and include only required S3 modules (#57629)
add 1a0309b12fd branch-3.1: [mem](jemalloc) Improve memory reuse efficiency and reduce page faults to change jemalloc conf #57076 (#57152)
add 0325b3597ca branch-3.1: [fix](decimal256) fix unstable overflow error when casting decimal256 to float #56848 (#57503)
add eb2b07df990 branch-3.1: [fix](filecache) fix filecache microbench bvar #56360 (#57535)
add de8ac0d1a34 branch-3.1: [feat](hdfs) upgrade libhdfs to 3.4.2 #57621 (#57638)
add 6a41b262e0d branch-3.1: [improve](glue) Glue endpoint is required. #57543 (#57610)
add 5be933095f6 branch-3.1: [fix](iceberg) Support client.region and iceberg.rest.* properties invended credentials #57521 (#57539)
add d64101b8d7a branch-3.1: [fix](docker) fix iceberg docker error: unable to prepare context: path "spark/" not found #57417 (#57611)
add e6f4a96a403 branch-3.1:[enhancement](s3)Add TotalGetRequestTime profile metric to the S3 reader. (#57519) (#57636)
add 873d39e96e9 branch-3.1:[enhancement](parquet)Optimize the performance of parquet reader when decode RLE_DICTIONARY encoding (#57208) (#57614)
add f5f1a6c61b5 branch-3.1: [Bug](Jni)should delete local ref after convert to global ref #57513 (#57597)
add 3d15cc61fe8 branch-3.1: [fix](cloud) Fix domain user set default cluster report err #56032 (#57555)
add c76cc0815aa branch-3.1: [chore](dep)upgrade dependencies #57402 (#57661)
add 10311bc3ef8 branch-3.1: [feat](catalog) Support catalog attribute connectivity tester (#57637)
add 206ad72d9e4 branch-3.1: [fix](build): Fix build.sh (#57684)
add fb43dc54470 branch-3.1: [chore](dep)upgrade netty and hadoop #57226 (#57608)
add 0852d806398 branch-3.1: [fix](schema_change) fix incorrect cache usage of schema change expr leading to schema errors #56602 (#57517)
add 00d81c8d6e3 branch-3.1: [fix](scanner) Make Scanner close() method thread-safe using atomic operations #57436 (#57644)
add 3bcf0c85131 branch-3.1: [fix] (parquet-reader) Fix parquet all row groups are filtered. #57490 (#57646)
add d47151ca741 branch-3.1: [fix](inverted index) fix match without inverted index #57468 (#57673)
add c0fe6bc3d40 branch-3.1: [fix](cloud) fix wrong private endpoint when only configured port number #57429 (#57675)
add 6b2adaa590b Merge branch 'branch-3.1' into auto-pick-57629-branch-3.1
No new revisions were added by this update.
Summary of changes:
be/CMakeLists.txt | 2 +-
be/src/io/cache/block_file_cache.h | 4 -
.../io/fs/connectivity/s3_connectivity_tester.cpp | 53 ++++
.../s3_connectivity_tester.h} | 13 +-
.../connectivity/storage_connectivity_tester.cpp} | 28 +-
.../storage_connectivity_tester.h} | 16 +-
be/src/io/fs/hdfs.h | 2 +-
be/src/io/fs/s3_file_reader.cpp | 12 +-
be/src/io/fs/s3_file_reader.h | 1 +
be/src/io/tools/file_cache_microbench.cpp | 8 +-
be/src/service/backend_service.cpp | 7 +
be/src/service/backend_service.h | 3 +
be/src/util/jni-util.cpp | 1 +
be/src/vec/data_types/data_type_decimal.h | 9 +-
be/src/vec/exec/format/parquet/decoder.cpp | 12 +-
.../format/parquet/fix_length_dict_decoder.hpp | 64 ++++-
be/src/vec/exec/scan/new_es_scanner.cpp | 2 +-
be/src/vec/exec/scan/new_jdbc_scanner.cpp | 3 +
be/src/vec/exec/scan/new_olap_scanner.cpp | 2 +-
be/src/vec/exec/scan/vfile_scanner.cpp | 13 +-
be/src/vec/exec/scan/vmeta_scanner.cpp | 5 +-
be/src/vec/exec/scan/vscan_node.h | 2 +-
be/src/vec/exec/scan/vscanner.cpp | 12 +-
be/src/vec/exec/scan/vscanner.h | 8 +-
be/src/vec/functions/match.cpp | 6 +-
.../parquet/fix_length_dict_decoder_test.cpp | 4 +-
build.sh | 132 ++++-----
conf/be.conf | 2 +-
.../docker-compose/iceberg/iceberg.yaml.tpl | 1 -
.../iceberg-metadata-scanner/pom.xml | 1 -
fe/be-java-extensions/preload-extensions/pom.xml | 98 +------
fe/fe-core/pom.xml | 27 --
.../java/org/apache/doris/alter/RollupJobV2.java | 11 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 10 +-
.../apache/doris/datasource/ExternalCatalog.java | 15 +
.../AWSGlueMetaStoreBaseConnectivityTester.java | 70 +++++
.../AbstractHiveConnectivityTester.java} | 23 +-
.../AbstractIcebergConnectivityTester.java} | 21 +-
.../AbstractS3CompatibleConnectivityTester.java | 69 +++++
.../CatalogConnectivityTestCoordinator.java | 320 +++++++++++++++++++++
.../connectivity/HMSBaseConnectivityTester.java | 62 ++++
.../HdfsCompatibleConnectivityTester.java | 51 ++++
.../connectivity/HdfsConnectivityTester.java} | 9 +-
.../HiveGlueMetaStoreConnectivityTester.java} | 30 +-
.../connectivity/HiveHMSConnectivityTester.java} | 29 +-
.../IcebergGlueMetaStoreConnectivityTester.java} | 30 +-
.../IcebergHMSConnectivityTester.java} | 29 +-
.../IcebergRestConnectivityTester.java | 85 ++++++
...cebergS3TablesMetaStoreConnectivityTester.java} | 21 +-
.../connectivity/MetaConnectivityTester.java} | 24 +-
.../connectivity/MinioConnectivityTester.java} | 21 +-
.../connectivity/S3ConnectivityTester.java} | 17 +-
.../connectivity/StorageConnectivityTester.java | 99 +++++++
.../datasource/credentials/CredentialUtils.java | 16 +-
.../metastore/AWSGlueMetaStoreBaseProperties.java | 71 ++++-
.../metastore/AbstractIcebergProperties.java | 1 +
.../metastore/HiveGlueMetaStoreProperties.java | 2 +
.../property/metastore/HiveHMSProperties.java | 3 +-
.../metastore/IcebergGlueMetaStoreProperties.java | 2 +
.../metastore/IcebergHMSMetaStoreProperties.java | 2 +
.../property/metastore/IcebergRestProperties.java | 2 +
.../property/storage/HdfsProperties.java | 17 ++
.../datasource/property/storage/S3Properties.java | 5 +-
.../property/storage/StorageProperties.java | 3 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 3 +-
.../apache/doris/mysql/privilege/UserProperty.java | 18 +-
.../WorkloadConditionUsername.java | 2 +-
.../hadoop/hbase/io/FSDataInputStreamWrapper.java | 3 +-
.../org/apache/doris/common/GenericPoolTest.java | 6 +
.../AWSGlueMetaStoreBasePropertiesTest.java | 2 +-
.../apache/doris/utframe/MockedBackendFactory.java | 9 +
fe/pom.xml | 81 ++----
gensrc/thrift/BackendService.thrift | 12 +
.../analyzer/test_custom_analyzer.out | 45 +++
.../cloud_p0/auth/test_set_default_cluster.groovy | 15 +
.../test_iceberg_rest_minio_connectivity.groovy | 105 +++++++
.../test_connection/test_connectivity.groovy | 207 +++++++++++++
.../analyzer/test_custom_analyzer.groovy | 77 +++++
.../test_alter_rollup_table.groovy | 136 +++++++++
79 files changed, 1961 insertions(+), 483 deletions(-)
create mode 100644 be/src/io/fs/connectivity/s3_connectivity_tester.cpp
copy be/src/io/fs/{encrypted_fs_factory.h => connectivity/s3_connectivity_tester.h} (77%)
copy be/src/{olap/rowset/segment_v2/inverted_index/util/string_helper.h => io/fs/connectivity/storage_connectivity_tester.cpp} (61%)
copy be/src/io/fs/{encrypted_fs_factory.h => connectivity/storage_connectivity_tester.h} (76%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AWSGlueMetaStoreBaseConnectivityTester.java
copy fe/fe-core/src/main/java/org/apache/doris/{analysis/DropAnalyzeJobStmt.java => datasource/connectivity/AbstractHiveConnectivityTester.java} (64%)
copy fe/fe-core/src/main/java/org/apache/doris/{analysis/UnsupportedStmt.java => datasource/connectivity/AbstractIcebergConnectivityTester.java} (60%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/CatalogConnectivityTestCoordinator.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/HMSBaseConnectivityTester.java
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/HdfsCompatibleConnectivityTester.java
copy fe/fe-core/src/main/java/org/apache/doris/{fs/remote/dfs/OFSFileSystem.java => datasource/connectivity/HdfsConnectivityTester.java} (77%)
copy fe/fe-core/src/main/java/org/apache/doris/{cloud/catalog/CloudDynamicPartitionProperty.java => datasource/connectivity/HiveGlueMetaStoreConnectivityTester.java} (51%)
copy fe/fe-core/src/main/java/org/apache/doris/{cloud/catalog/CloudDynamicPartitionProperty.java => datasource/connectivity/HiveHMSConnectivityTester.java} (55%)
copy fe/fe-core/src/main/java/org/apache/doris/{cloud/catalog/CloudDynamicPartitionProperty.java => datasource/connectivity/IcebergGlueMetaStoreConnectivityTester.java} (51%)
copy fe/fe-core/src/main/java/org/apache/doris/{cloud/catalog/CloudDynamicPartitionProperty.java => datasource/connectivity/IcebergHMSConnectivityTester.java} (54%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTester.java
copy fe/{fe-common/src/main/java/org/apache/doris/cloud/security/DummySecurityChecker.java => fe-core/src/main/java/org/apache/doris/datasource/connectivity/IcebergS3TablesMetaStoreConnectivityTester.java} (62%)
copy fe/fe-core/src/main/java/org/apache/doris/{analysis/ImportColumnsStmt.java => datasource/connectivity/MetaConnectivityTester.java} (65%)
copy fe/fe-core/src/main/java/org/apache/doris/{analysis/DropAnalyzeJobStmt.java => datasource/connectivity/MinioConnectivityTester.java} (68%)
copy fe/fe-core/src/main/java/org/apache/doris/{analysis/ImportDeleteOnStmt.java => datasource/connectivity/S3ConnectivityTester.java} (68%)
create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/StorageConnectivityTester.java
create mode 100644 regression-test/suites/external_table_p0/test_connection/test_iceberg_rest_minio_connectivity.groovy
create mode 100644 regression-test/suites/external_table_p2/test_connection/test_connectivity.groovy
create mode 100644 regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org