You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/06/01 15:02:05 UTC
(asterixdb) 19/19: Merge branch 'gerrit/ionic' into 'master'
This is an automated email from the ASF dual-hosted git repository.
mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ba3661faf669fd90cc35c260505e9f03dcd9c8ed
Merge: bab942f7c4 167e58a69c
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat May 31 15:58:55 2025 -0400
Merge branch 'gerrit/ionic' into 'master'
Change-Id: Idaae7d6906e39513691a35bd243286638660e72d
.../InjectTypeCastForFunctionArgumentsRule.java | 1 +
.../asterix/translator/CompiledStatements.java | 10 +-
.../translator/LangExpressionToPlanTranslator.java | 2 +-
.../SqlppExpressionToPlanTranslator.java | 1 +
asterixdb/asterix-app/pom.xml | 2 +-
.../hash_join_exchange.5.query.sqlpp | 47 +++++
.../hash_join_exchange.7.query.sqlpp} | 16 +-
.../hash_join_exchange.9.query.sqlpp} | 27 ++-
.../hash_join_exchange_1.11.query.sqlpp} | 29 ++-
.../hash_join_exchange_1.13.query.sqlpp | 36 ++++
.../hash_join_exchange_1.5.query.sqlpp} | 32 ++-
.../hash_join_exchange_1.7.query.sqlpp} | 21 +-
.../hash_join_exchange_1.9.query.sqlpp} | 20 +-
.../hash_join_exchange.5.query.plan | 40 ++++
.../hash_join_exchange.7.query.plan | 40 ++++
.../hash_join_exchange.9.query.plan | 54 +++++
.../hash_join_exchange_1.11.query.plan | 38 ++++
.../hash_join_exchange_1.13.query.plan | 36 ++++
.../hash_join_exchange_1.5.query.plan | 36 ++++
.../hash_join_exchange_1.7.query.plan | 38 ++++
.../hash_join_exchange_1.9.query.plan | 38 ++++
.../copy-to/csv/datetime/test.000.ddl.sqlpp} | 12 +-
.../copy-to/csv/datetime/test.030.update.sqlpp} | 29 +--
.../copy-to/csv/datetime/test.031.update.sqlpp} | 30 +--
.../copy-to/csv/datetime/test.032.update.sqlpp} | 30 +--
.../copy-to/csv/datetime/test.033.update.sqlpp} | 31 +--
.../copy-to/csv/datetime/test.034.update.sqlpp} | 30 +--
.../copy-to/csv/datetime/test.035.update.sqlpp} | 30 +--
.../copy-to/csv/datetime/test.036.update.sqlpp} | 32 +--
.../copy-to/csv/datetime/test.040.ddl.sqlpp | 107 ++++++++++
.../copy-to/csv/datetime/test.130.query.sqlpp} | 18 +-
.../copy-to/csv/datetime/test.131.query.sqlpp} | 16 +-
.../copy-to/csv/datetime/test.132.query.sqlpp} | 17 +-
.../copy-to/csv/datetime/test.133.query.sqlpp} | 16 +-
.../copy-to/csv/datetime/test.134.query.sqlpp} | 16 +-
.../copy-to/csv/datetime/test.135.query.sqlpp} | 16 +-
.../copy-to/csv/datetime/test.136.query.sqlpp} | 18 +-
.../copy-to/csv/datetime/test.999.ddl.sqlpp} | 12 +-
.../queries_sqlpp/crypto/sha1/sha1.01.ddl.sqlpp} | 17 +-
.../crypto/sha1/sha1.02.update.sqlpp} | 21 +-
.../queries_sqlpp/crypto/sha1/sha1.03.query.sqlpp} | 16 +-
.../queries_sqlpp/crypto/sha1/sha1.04.query.sqlpp} | 16 +-
.../queries_sqlpp/crypto/sha1/sha1.99.ddl.sqlpp} | 14 +-
.../hash_join_exchange.1.ddl.sqlpp} | 14 +-
.../hash_join_exchange.2.update.sqlpp | 30 +++
.../hash_join_exchange.3.update.sqlpp | 37 ++++
.../hash_join_exchange.4.query.sqlpp} | 19 +-
.../hash_join_exchange.6.query.sqlpp} | 20 +-
.../hash_join_exchange.8.query.sqlpp} | 19 +-
.../hash_join_exchange_1.1.ddl.sqlpp} | 13 +-
.../hash_join_exchange_1.10.query.sqlpp} | 21 +-
.../hash_join_exchange_1.12.query.sqlpp | 28 +++
.../hash_join_exchange_1.2.update.sqlpp} | 16 +-
.../hash_join_exchange_1.3.update.sqlpp} | 15 +-
.../hash_join_exchange_1.4.query.sqlpp} | 21 +-
.../hash_join_exchange_1.6.query.sqlpp} | 19 +-
.../hash_join_exchange_1.8.query.sqlpp} | 18 +-
.../handle_error_fun.01.query.sqlpp} | 21 +-
.../numeric/from_base/from_base.01.ddl.sqlpp} | 17 +-
.../numeric/from_base/from_base.02.update.sqlpp} | 25 ++-
.../numeric/from_base/from_base.03.query.sqlpp} | 16 +-
.../numeric/from_base/from_base.04.query.sqlpp} | 18 +-
.../numeric/from_base/from_base.05.query.sqlpp} | 15 +-
.../numeric/from_base/from_base.99.ddl.sqlpp} | 14 +-
.../string/pad/lpad/lpad.00.ddl.sqlpp} | 20 +-
.../string/pad/lpad/lpad.01.query.sqlpp} | 19 +-
.../string/pad/lpad/lpad.02.query.sqlpp} | 19 +-
.../string/pad/lpad/lpad.03.query.sqlpp} | 16 +-
.../string/pad/rpad/rpad.00.ddl.sqlpp} | 20 +-
.../string/pad/rpad/rpad.01.query.sqlpp} | 19 +-
.../string/pad/rpad/rpad.02.query.sqlpp} | 18 +-
.../string/pad/rpad/rpad.03.query.sqlpp} | 16 +-
.../results/copy-to/csv/datetime/result.130.adm | 1 +
.../results/copy-to/csv/datetime/result.131.adm | 1 +
.../results/copy-to/csv/datetime/result.132.adm | 1 +
.../results/copy-to/csv/datetime/result.133.adm | 1 +
.../results/copy-to/csv/datetime/result.134.adm | 1 +
.../results/copy-to/csv/datetime/result.135.adm | 1 +
.../results/copy-to/csv/datetime/result.136.adm | 1 +
.../runtimets/results/crypto/sha1/sha1.03.adm | 6 +
.../runtimets/results/crypto/sha1/sha1.04.adm | 1 +
.../hash_join_exchange/hash_join_exchange.4.adm | 10 +
.../hash_join_exchange/hash_join_exchange.6.adm | 10 +
.../hash_join_exchange/hash_join_exchange.8.adm | 10 +
.../hash_join_exchange_1.10.adm | 100 +++++++++
.../hash_join_exchange_1.12.adm | 100 +++++++++
.../hash_join_exchange_1.4.adm | 100 +++++++++
.../hash_join_exchange_1.6.adm | 100 +++++++++
.../hash_join_exchange_1.8.adm | 100 +++++++++
.../misc/handle_error_fun/handle_error_fun.01.adm | 1 +
.../results/numeric/from_base/from_base.03.adm | 8 +
.../results/numeric/from_base/from_base.04.adm | 1 +
.../results/numeric/from_base/from_base.05.adm | 1 +
.../runtimets/results/string/pad/lpad/lpad.01.adm | 5 +
.../runtimets/results/string/pad/lpad/lpad.02.adm | 1 +
.../runtimets/results/string/pad/lpad/lpad.03.adm | 1 +
.../runtimets/results/string/pad/rpad/rpad.01.adm | 5 +
.../runtimets/results/string/pad/rpad/rpad.02.adm | 1 +
.../runtimets/results/string/pad/rpad/rpad.03.adm | 1 +
.../src/test/resources/runtimets/sqlpp_queries.xml | 37 ++++
.../runtimets/testsuite_external_dataset_s3.xml | 10 +
.../apache/asterix/cloud/clients/ICloudClient.java | 8 +
.../asterix/cloud/clients/NoopCloudClient.java | 6 +
.../asterix/cloud/clients/UnstableCloudClient.java | 6 +
.../cloud/clients/aws/s3/S3CloudClient.java | 6 +
.../blobstorage/AzBlobStorageCloudClient.java | 6 +
.../cloud/clients/google/gcs/GCSCloudClient.java | 6 +
.../parquet/ParquetSchemaInferPoolWriter.java | 8 +
.../writer/AbstractCloudExternalFileWriter.java | 19 +-
.../AbstractCloudExternalFileWriterFactory.java | 25 ++-
.../common/context/GlobalVirtualBufferCache.java | 4 +-
asterixdb/asterix-external-data/pom.xml | 5 -
.../printer/parquet/ParquetRecordLazyVisitor.java | 20 +-
.../printer/parquet/ParquetSchemaLazyVisitor.java | 17 +-
.../writer/printer/parquet/ParquetSchemaTree.java | 35 ++++
.../lang/common/statement/CopyToStatement.java | 20 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 15 +-
.../metadata/declared/IExternalWriteDataSink.java | 4 +
.../asterix/metadata/declared/WriteDataSink.java | 9 +-
.../metadata/provider/ExternalWriterProvider.java | 3 +-
.../data/nontagged/printers/PrintTools.java | 20 +-
.../printers/csv/ADatePrinterFactory.java | 8 +-
.../printers/csv/ADateTimePrinterFactory.java | 8 +-
.../printers/csv/AObjectPrinterFactory.java | 109 +++++++++-
.../printers/csv/ARecordPrinterFactory.java | 7 +-
.../printers/csv/ATimePrinterFactory.java | 8 +-
.../nontagged/CSVPrinterFactoryProvider.java | 14 +-
.../asterix/om/exceptions/ExceptionUtil.java | 9 +
.../asterix/om/functions/BuiltinFunctions.java | 19 +-
.../om/pointables/printer/csv/APrintVisitor.java | 7 +-
.../evaluators/functions/FromBaseDescriptor.java | 127 ++++++++++++
.../evaluators/functions/IfErrorDescriptor.java | 70 +++++++
.../evaluators/functions/Sha1Base64Descriptor.java | 57 +++++
.../evaluators/functions/Sha1HexDescriptor.java | 56 +++++
.../evaluators/functions/Sha1StringEvaluator.java | 111 ++++++++++
.../evaluators/functions/StringLpadDescriptor.java | 56 +++++
.../evaluators/functions/StringPadEvaluator.java | 169 +++++++++++++++
.../evaluators/functions/StringRpadDescriptor.java | 56 +++++
.../runtime/functions/FunctionCollection.java | 14 ++
asterixdb/pom.xml | 229 ++-------------------
.../physical/AbstractHashJoinPOperator.java | 25 +--
.../org/apache/hyracks/api/util/InvokeUtil.java | 82 ++++----
.../context/DefaultCloudReadContext.java | 3 +-
.../cloud/io/request/ICloudBeforeRetryRequest.java | 3 +-
.../cloud/io/request/ICloudRetryPredicate.java | 19 +-
.../cloud/util/CloudRetryableRequestUtil.java | 48 ++++-
.../data/std/primitive/UTF8StringPointable.java | 19 ++
hyracks-fullstack/hyracks/hyracks-hdfs/pom.xml | 4 +-
.../am/btree/impls/BTreeCountingSearchCursor.java | 4 +-
.../am/btree/impls/BTreeRangeSearchCursor.java | 17 +-
.../hyracks/storage/am/btree/impls/DiskBTree.java | 2 +-
.../am/btree/impls/DiskBTreePointSearchCursor.java | 1 -
.../am/btree/impls/DiskBTreeRangeSearchCursor.java | 3 +-
.../common/impls/TreeIndexDiskOrderScanCursor.java | 5 +-
.../lsm/tuples/ColumnMultiBufferProvider.java | 4 +-
.../impls/MultitenantVirtualBufferCache.java | 4 +-
.../am/lsm/common/impls/VirtualBufferCache.java | 4 +-
.../impls/AbstractOnDiskInvertedListCursor.java | 1 +
.../FixedSizeElementInvertedListScanCursor.java | 2 +-
.../storage/common/buffercache/BufferCache.java | 6 +-
.../common/buffercache/DebugBufferCache.java | 4 +-
.../storage/common/buffercache/IBufferCache.java | 4 +-
.../storage/am/btree/StorageFileAccessTest.java | 38 ++--
.../am/lsm/btree/impl/TestVirtualBufferCache.java | 4 +-
hyracks-fullstack/pom.xml | 95 ++++++++-
165 files changed, 3001 insertions(+), 1063 deletions(-)
diff --cc asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
index fb83aa98a4,0000000000..8f37191b44
mode 100644,000000..100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/NoopCloudClient.java
@@@ -1,183 -1,0 +1,189 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.cloud.clients;
+
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
++import java.util.function.Predicate;
+
+import org.apache.asterix.cloud.IWriteBufferProvider;
+import org.apache.asterix.cloud.clients.profiler.IRequestProfilerLimiter;
+import org.apache.asterix.cloud.clients.profiler.NoOpRequestProfilerLimiter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.control.nc.io.IOManager;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+public class NoopCloudClient implements ICloudClient {
+
+ public static final ICloudClient INSTANCE = new NoopCloudClient();
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private NoopCloudClient() {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // no-op
+ }
+
+ @Override
+ public int getWriteBufferSize() {
+ return 0;
+ }
+
+ @Override
+ public IRequestProfilerLimiter getProfilerLimiter() {
+ return NoOpRequestProfilerLimiter.INSTANCE;
+ }
+
+ @Override
+ public ICloudWriter createWriter(String bucket, String path, IWriteBufferProvider bufferProvider) {
+ return new NoOpCloudWriter();
+ }
+
+ @Override
+ public Set<CloudFile> listObjects(String bucket, String path, FilenameFilter filter) {
+ return Set.of();
+ }
+
+ @Override
+ public int read(String bucket, String path, long offset, ByteBuffer buffer) throws HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public byte[] readAllBytes(String bucket, String path) throws HyracksDataException {
+ return new byte[0];
+ }
+
+ @Override
+ public InputStream getObjectStream(String bucket, String path, long offset, long length) {
+ return InputStream.nullInputStream();
+ }
+
+ @Override
+ public void write(String bucket, String path, byte[] data) {
+ }
+
+ @Override
+ public void copy(String bucket, String srcPath, FileReference destPath) {
+ }
+
+ @Override
+ public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
+ }
+
+ @Override
+ public long getObjectSize(String bucket, String path) throws HyracksDataException {
+ return 0;
+ }
+
+ @Override
+ public boolean exists(String bucket, String path) throws HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public boolean isEmptyPrefix(String bucket, String path) throws HyracksDataException {
+ return false;
+ }
+
+ @Override
+ public IParallelDownloader createParallelDownloader(String bucket, IOManager ioManager)
+ throws HyracksDataException {
+ return NoOpParallelDownloader.INSTANCE;
+ }
+
++ @Override
++ public Predicate<Exception> getObjectNotFoundExceptionPredicate() {
++ return e -> true;
++ }
++
+ @Override
+ public JsonNode listAsJson(ObjectMapper objectMapper, String bucket) {
+ return OBJECT_MAPPER.createArrayNode();
+ }
+
+ private static class NoOpCloudWriter implements ICloudWriter {
+
+ long position = 0L;
+
+ public NoOpCloudWriter() {
+ }
+
+ @Override
+ public void abort() throws HyracksDataException {
+ position = 0L;
+ }
+
+ @Override
+ public int write(ByteBuffer page) throws HyracksDataException {
+ int written = page.remaining();
+ position += written;
+ page.position(page.limit());
+ return written;
+ }
+
+ @Override
+ public void write(int b) throws HyracksDataException {
+ position++;
+ }
+
+ @Override
+ public int write(byte[] b, int off, int len) throws HyracksDataException {
+ position += len;
+ return len;
+ }
+
+ @Override
+ public long position() {
+ return position;
+ }
+
+ @Override
+ public void finish() throws HyracksDataException {
+ position = 0;
+ }
+ }
+
+ private static class NoOpParallelDownloader implements IParallelDownloader {
+ private static final NoOpParallelDownloader INSTANCE = new NoOpParallelDownloader();
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void downloadFiles(Collection<FileReference> toDownload) throws HyracksDataException {
+ }
+
+ @Override
+ public Collection<FileReference> downloadDirectories(Collection<FileReference> toDownload)
+ throws HyracksDataException {
+ return List.of();
+ }
+ }
+}
diff --cc asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
index fc35792402,1195b8cfee..8a28f53edf
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3CloudClient.java
@@@ -33,7 -34,7 +33,8 @@@ import java.util.HashSet
import java.util.Iterator;
import java.util.List;
import java.util.Set;
+import java.util.function.Function;
+ import java.util.function.Predicate;
import org.apache.asterix.cloud.CloudResettableInputStream;
import org.apache.asterix.cloud.IWriteBufferProvider;
diff --cc asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index 7922fc4dd5,91a4fbae9d..e2bbb5b0f2
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@@ -391,15 -391,21 +392,20 @@@ public class AzBlobStorageCloudClient i
// Hence this implementation is a no op.
}
+ @Override
+ public Predicate<Exception> getObjectNotFoundExceptionPredicate() {
+ return ex -> (ex instanceof BlobStorageException bse) && bse.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND);
+ }
+
- private static BlobContainerClient buildClient(AzBlobStorageClientConfig config) {
- BlobContainerClientBuilder blobContainerClientBuilder =
- new BlobContainerClientBuilder().containerName(config.getBucket()).endpoint(getEndpoint(config));
- configCredentialsToAzClient(blobContainerClientBuilder, config);
- BlobContainerClient blobContainerClient = blobContainerClientBuilder.buildClient();
- blobContainerClient.createIfNotExists();
- return blobContainerClient;
+ private static BlobServiceClient buildClient(AzBlobStorageClientConfig config) {
+ BlobServiceClientBuilder blobServiceClientBuilder = new BlobServiceClientBuilder();
+ blobServiceClientBuilder.endpoint(getEndpoint(config));
+ blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
+ configCredentialsToAzClient(blobServiceClientBuilder, config);
+ return blobServiceClientBuilder.buildClient();
}
- private static void configCredentialsToAzClient(BlobContainerClientBuilder builder,
+ private static void configCredentialsToAzClient(BlobServiceClientBuilder builder,
AzBlobStorageClientConfig config) {
if (config.isAnonymousAuth()) {
StorageSharedKeyCredential creds =
diff --cc asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
index 977f90a1a7,2007e547dd..c52e2b61c6
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriterFactory.java
@@@ -119,10 -117,11 +121,10 @@@ abstract class AbstractCloudExternalFil
if (staticPath != null) {
if (isExceedingMaxLength(staticPath, S3ExternalFileWriter.MAX_LENGTH_IN_BYTES)) {
throw new CompilationException(ErrorCode.WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH, pathSourceLocation,
- staticPath, S3ExternalFileWriter.MAX_LENGTH_IN_BYTES,
- ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
+ staticPath, getPathMaxLengthInBytes(), getAdapterName());
}
- if (!testClient.isEmptyPrefix(bucket, staticPath)) {
+ if (!runWithNoRetryOnInterruption(() -> testClient.isEmptyPrefix(bucket, staticPath))) {
// Ensure that the static path is empty
throw new CompilationException(ErrorCode.DIRECTORY_IS_NOT_EMPTY, pathSourceLocation, staticPath);
}