You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2025/11/06 17:28:17 UTC

(asterixdb) branch master updated (f388dbc8d9 -> 362f97b98a)

This is an automated email from the ASF dual-hosted git repository.

mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


    from f388dbc8d9 [ASTERIXDB-3635][COMP] Fix array skip-index annotation
     new 95d55701d4 [ASTERIXDB-3635][COMP] Tune single dataset index costing
     new 75dd626ac1 [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
     new 74652bf3f3 Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
     new 9e12be5e57 [NO ISSUE][EXT]: Add method to generate an external ID
     new b9d29911b4 [NO ISSUE]: Refactor AWS common properties
     new 69840d4929 [NO ISSUE][*DB][COMMON] Reduce logging on dataset async flush batches
     new 44adbba68d [NO ISSUE][*DB][STO] Increased log brevity in cloud storage
     new 384570e475 [ASTERIXDB-3392][EXT] Default to STRING for null-typed fields during schema inference in COPY TO parquet
     new 4874143e7d [NO ISSUE][*DB][EXT] Make IDataParser extend Closeable, misc
     new 909f6ce5c9 Merge branch 'gerrit/neo' into 'gerrit/trinity'
     new 48509e02c4 [NO ISSUE][EXT]: Use ABFSS instead of deprecated WASBS
     new b20e3c66d9 [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
     new e27f6f95ad Merge branch 'gerrit/neo' into 'gerrit/trinity'
     new 95f77257b4 Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
     new 0c58b35854 [ASTERIXDB-3392][EXT] Fix Hdfs copy to parquet test failures
     new 4e0194dc1f [NO ISSUE][RT] Respond to NC in case of unexpected failures
     new b67dac4236 [NO ISSUE][*DB][COMMON] Fix async checkpoint failure if only metadata is flushed
     new 3050d373ed [ASTERIXDB-3657][FAIL]: handle non-serializable exceptions
     new 4cb134ee3d [NO ISSUE][CLOUD] Updating deleteObjects for AzureBlob
     new 8c18a52aa2 [ASTERIXDB-3648][COMP] Fix incorrect replacement of Left Outer Joins with Unnest in CBO
     new 6589517084 [ASTERIXDB-3649][*DB] Improve async request API
     new c8542b55f0 [ASTERIXDB-3650][COMP] Ignore sample indexes when checking for secondary indexes
     new d86e2327d1 [NO ISSUE][*DB][STO] Cloud, logging misc improvements
     new 39ac77d88a [NO ISSUE][EXT]: change azure cloud client to accept both clients
     new 362f97b98a Merge branch 'gerrit/phoenix' into 'master'

The 25 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../org/apache/asterix/active/ActiveManager.java   |   1 +
 .../optimizer/rules/PushFieldAccessRule.java       |  13 +-
 .../optimizer/rules/cbo/EnumerateJoinsRule.java    |  14 +-
 .../asterix/optimizer/rules/cbo/JoinNode.java      | 107 ++--
 .../apache/asterix/translator/ClientRequest.java   |   4 +
 .../asterix/translator/IStatementExecutor.java     |  33 +-
 .../apache/asterix/translator/ResultMetadata.java  |  38 +-
 .../apache/asterix/translator/SessionConfig.java   |  21 +-
 .../apache/asterix/api/common/APIFramework.java    |  13 +-
 .../api/http/server/AbstractQueryApiServlet.java   | 195 +++++++
 .../api/http/server/NCQueryResultApiServlet.java   |  61 +++
 .../api/http/server/NCQueryServiceServlet.java     |   7 +-
 .../api/http/server/NCQueryStatusApiServlet.java   |  20 +-
 .../api/http/server/QueryResultApiServlet.java     | 141 +++--
 .../http/server/QueryServiceRequestParameters.java |  14 +-
 .../api/http/server/QueryServiceServlet.java       | 156 +-----
 .../api/http/server/QueryStatusApiServlet.java     | 115 +++-
 .../apache/asterix/api/http/server/ResultUtil.java |  21 +
 .../asterix/api/http/server/StorageApiServlet.java |  11 +-
 .../external/ExternalCredentialsCacheUpdater.java  |   4 +-
 .../message/AbstractInternalRequestMessage.java    |   2 +
 .../app/message/ClientInfoRequestMessage.java      |  67 +++
 .../app/message/ClientInfoResponseMessage.java     |  50 ++
 .../DiscardResultPartitionRequestMessage.java      |  57 ++
 .../app/message/DiscardResultRequestMessage.java   |  62 +++
 .../app/message/DiscardResultResponseMessage.java  |  44 ++
 .../message/ExecuteStatementRequestMessage.java    |  32 +-
 .../app/message/StorageCleanupRequestMessage.java  |   4 +-
 .../apache/asterix/app/nc/NCAppRuntimeContext.java |   4 +-
 .../org/apache/asterix/app/nc/RecoveryManager.java |  34 +-
 .../org/apache/asterix/app/nc/ReplicaManager.java  |  37 +-
 .../nc/task/CloudToLocalStorageCachingTask.java    |  14 +-
 .../asterix/app/nc/task/LocalRecoveryTask.java     |  12 +-
 .../app/nc/task/LocalStorageCleanupTask.java       |   5 +-
 .../asterix/app/nc/task/UpdateNodeStatusTask.java  |   9 +-
 .../app/replication/NcLifecycleCoordinator.java    |  13 +-
 .../message/NCLifecycleTaskReportMessage.java      |  11 +-
 .../message/RegistrationTasksRequestMessage.java   |  12 +-
 .../message/RegistrationTasksResponseMessage.java  |   5 +-
 .../asterix/app/result/JobResultCallback.java      |   3 +
 .../apache/asterix/app/result/ResultHandle.java    |  75 ++-
 .../apache/asterix/app/result/ResultReader.java    |  10 +
 .../app/result/fields/CreatedAtPrinter.java}       |  35 +-
 .../asterix/app/result/fields/MetricsPrinter.java  | 151 ++++--
 .../asterix/app/result/fields/NcResultPrinter.java |  12 +-
 .../app/result/fields/PartitionInfoPrinter.java    |  64 +++
 .../app/result/fields/ResultCountPrinter.java}     |  32 +-
 .../asterix/app/translator/QueryTranslator.java    | 152 ++++--
 .../asterix/hyracks/bootstrap/CCApplication.java   |   9 +
 .../asterix/hyracks/bootstrap/NCApplication.java   |  17 +-
 .../apache/asterix/messaging/CCMessageBroker.java  |   9 +
 .../apache/asterix/utils/AsyncRequestsAPIUtil.java | 151 ++++++
 .../org/apache/asterix/utils/RedactionUtil.java    |   2 +-
 .../asterix/runtime/ClusterStateManagerTest.java   |   7 +-
 .../asterix/test/common/ResultExtractor.java       |   7 +-
 .../apache/asterix/test/common/TestExecutor.java   |  55 +-
 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 ++++
 .../async-compilation-failed.1.async.sqlpp         |   1 -
 .../async-exhausted-result.1.async.sqlpp           |   1 -
 .../async-exhausted-result.2.pollget.http}         |   6 +-
 .../async-exhausted-result.3.get.http}             |   0
 .../async-exhausted-result.4.delete.http}          |   0
 .../async-exhausted-result.5.get.http}             |   0
 .../async-failed/async-failed.1.async.sqlpp        |   1 -
 .../async-failed/async-failed.2.pollget.http}      |   0
 .../async-json/async-json.1.async.sqlpp            |   1 -
 .../async-json/async-json.2.pollget.http}          |   0
 .../async-json/async-json.3.get.http}              |   0
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 -
 .../async-repeated/async-repeated.2.pollget.http}  |   0
 .../async-repeated/async-repeated.3.get.http}      |   0
 .../async-repeated/async-repeated.4.get.http}      |   0
 .../async-repeated/async-repeated.5.query.sqlpp    |   0
 .../async-running/async-running.1.async.sqlpp      |   1 -
 .../async-running/async-running.2.pollget.http}    |   0
 .../async-running/async-running.3.pollget.http}    |   0
 .../async-running/async-running.4.get.http}        |   0
 .../async-timeout}/async.1.async.sqlpp             |   4 +-
 .../async-timeout/async.2.pollget.http}            |   0
 .../async/async.1.async.sqlpp                      |   1 -
 .../async/async.2.pollget.http}                    |   0
 .../async/async.3.get.http}                        |   0
 .../deferred/deferred.1.deferred.sqlpp}            |   3 +-
 .../deferred/deferred.2.get.http}                  |   0
 .../async-deferred/AsyncDeferredQueries.xml        |  52 ++
 .../async-compilation-failed.1.async.sqlpp         |   1 +
 .../async-exhausted-result.1.async.sqlpp           |   1 +
 .../async-failed/async-failed.1.async.sqlpp        |   1 +
 .../async-json/async-json.1.async.sqlpp            |   1 +
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 +
 .../async-running/async-running.1.async.sqlpp      |   1 +
 .../async-deferred/async/async.1.async.sqlpp       |   1 +
 .../parquet-error-checks.23.ddl.sqlpp              |   1 +
 .../parquet-null1.01.ddl.sqlpp}                    |   3 +
 .../parquet-null1.02.update.sqlpp}                 |  25 +-
 .../parquet-null1.03.ddl.sqlpp}                    |  15 +-
 .../parquet-null1/parquet-null1.04.query.sqlpp}    |   7 +-
 .../parquet-null2.01.ddl.sqlpp}                    |   5 +-
 .../parquet-null2.02.update.sqlpp}                 |  17 +-
 .../parquet-null2.03.ddl.sqlpp}                    |  15 +-
 .../parquet-null2/parquet-null2.04.query.sqlpp}    |   7 +-
 .../parquet-simple/parquet-simple.01.ddl.sqlpp     |   2 +
 .../parquet-simple/parquet-simple.02.update.sqlpp  |   4 +-
 .../queries_sqlpp/flwor/at00/at00.6.async.sqlpp    |   1 +
 .../async-compilation-failed.1.regex               |   0
 .../async-exhausted-result.1.ignore                |   0
 .../async-exhausted-result.2.regexjson             |   8 +
 .../async-exhausted-result.3.regexjson             |  17 +
 .../async-exhausted-result.4.ignore                |  17 +
 .../async-failed/async-failed.1.ignore             |   0
 .../async-failed/async-failed.2.regex              |   0
 .../async-json/async-json.1.ignore                 |   0
 .../async-json/async-json.2.regexjson              |   8 +
 .../async-json/async-json.3.json                   |   0
 .../async-repeated/async-repeated.1.ignore         |   0
 .../async-repeated/async-repeated.2.regexjson      |   8 +
 .../async-repeated/async-repeated.3.regexjson      |  17 +
 .../async-repeated/async-repeated.4.regexjson      |  17 +
 .../async-repeated/async-repeated.5.json           |   0
 .../async-running/async-running.1.ignore           |   0
 .../async-running/async-running.2.json             |   0
 .../async-running/async-running.3.regexjson        |   8 +
 .../async-running/async-running.4.regexjson        |   8 +
 .../async-timeout/async-timeout.1.ignore}          |   0
 .../async-timeout/async-timeout.2.regex            |   3 +
 .../async/async.1.ignore                           |   0
 .../async/async.2.regexjson                        |   8 +
 .../async/async.3.regexjson                        |  17 +
 .../deferred/deferred.1.ignore                     |   0
 .../deferred/deferred.2.regexjson                  |  17 +
 .../copy-to/parquet-null1/parquet-null1.04.adm     |   2 +
 .../copy-to/parquet-null2/parquet-null2.04.adm     |   3 +
 .../single-collection-queries.3.plan               |  26 +-
 .../single-collection-queries.4.plan               |  26 +-
 .../single-collection-queries.7.plan               |  30 +-
 .../runtimets/testsuite_external_dataset_s3.xml    |  21 +-
 .../resources/runtimets/testsuite_sqlpp_hdfs.xml   |   1 -
 .../asterix/cloud/AbstractCloudIOManager.java      |  12 +-
 .../asterix/cloud/LocalPartitionBootstrapper.java  |   4 +-
 .../cloud/clients/aws/s3/S3ClientConfig.java       |  22 +-
 .../cloud/clients/aws/s3/S3ParallelDownloader.java |   2 +-
 .../blobstorage/AzBlobStorageClientConfig.java     |   2 +-
 .../blobstorage/AzBlobStorageCloudClient.java      | 101 ++--
 .../cloud/lazy/accessor/InitialCloudAccessor.java  |   5 +-
 .../lazy/accessor/ReplaceableCloudAccessor.java    |   5 +-
 .../lazy/accessor/SelectiveCloudAccessor.java      |   4 +-
 .../apache/asterix/cloud/util/CloudFileUtil.java   |   1 -
 .../writer/AzureExternalFileWriterFactory.java     |   2 +-
 .../cloud/writer/S3ExternalFileWriterFactory.java  |   4 +-
 .../org/apache/asterix/cloud/s3/LSMS3Test.java     |   2 +-
 .../lsm/flush/FlushColumnTupleWriter.java          |  14 +-
 .../apache/asterix/common/api/IRequestTracker.java |   8 +
 .../common/cloud/IPartitionBootstrapper.java       |   4 +-
 .../common/cluster/IClusterStateManager.java       |   3 +-
 .../cluster/StorageComputePartitionsMap.java       |   7 +-
 .../asterix/common/config/CloudProperties.java     |  75 +--
 .../asterix/common/config/MetadataProperties.java  |   4 +-
 .../asterix/common/config/PropertiesAccessor.java  |   7 +-
 .../common/context/DatasetLifecycleManager.java    |  19 +-
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 .../common/messaging/api/ICCMessageBroker.java     |   8 +
 .../asterix/common/storage/IReplicaManager.java    |  10 +-
 .../common/transactions/IRecoveryManager.java      |   8 +-
 .../org/apache/asterix/common/utils/Datasets.java  | 105 ++++
 .../asterix/common/utils/IntSortedBitSet.java      | 234 +++++++++
 .../org/apache/asterix/common/utils/IntUtil.java   | 119 +++++
 .../org/apache/asterix/common/utils/MathUtil.java  |  53 ++
 .../apache/asterix/common/utils/Partitions.java    | 240 +++++++++
 .../asterix/common/utils/ShortSortedBitSet.java    | 235 +++++++++
 .../org/apache/asterix/common/utils/ShortUtil.java | 122 +++++
 .../apache/asterix/test/utils/MathUtilTest.java    |  79 +++
 .../asterix/test/utils/ShortSortedBitSetTest.java  | 108 ++++
 .../apache/asterix/external/api/IDataParser.java   |  13 +-
 .../asterix/external/api/IRecordConverter.java     |   8 +-
 .../dataflow/FeedRecordDataFlowController.java     |   2 +
 .../external/input/HDFSDataSourceFactory.java      |   3 +-
 .../input/record/reader/aws/AwsS3InputStream.java  |   9 +-
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  |   2 +-
 .../aws/parquet/AwsS3ParquetReaderFactory.java     |   2 +-
 .../asterix/external/util/ExternalDataUtils.java   |  11 +-
 .../asterix/external/util/aws/AwsConstants.java    |  42 ++
 .../apache/asterix/external/util/aws/AwsUtils.java | 286 ++++++++++
 .../asterix/external/util/aws/s3/S3AuthUtils.java  | 577 ---------------------
 .../asterix/external/util/aws/s3/S3Constants.java  |  16 -
 .../asterix/external/util/aws/s3/S3Utils.java      | 313 ++++++++++-
 .../util/azure/blob_storage/AzureConstants.java    |   2 +-
 .../util/azure/blob_storage/AzureUtils.java        |  21 +-
 .../writer/HDFSExternalFileWriterFactory.java      |   2 +-
 .../writer/printer/parquet/ParquetSchemaTree.java  |  21 +-
 .../printer/parquet/SchemaCheckerLazyVisitor.java  |   6 +-
 .../input/record/reader/awss3/AwsS3Test.java       |   4 +-
 .../metadata/declared/MetadataProvider.java        |   8 +
 .../apache/asterix/metadata/utils/DatasetUtil.java |   6 +-
 .../replication/logging/RemoteLogsNotifier.java    |   7 +-
 .../replication/sync/ReplicaSynchronizer.java      |   4 +-
 .../asterix/runtime/utils/ClusterStateManager.java |  12 +-
 .../asterix/runtime/utils/RequestTracker.java      |  18 +
 .../PersistentLocalResourceRepository.java         |  11 +-
 asterixdb/pom.xml                                  |  16 +-
 ...ReplacementsAwareJavaSerializationProvider.java |  82 +++
 .../api/result/IResultPartitionManager.java        |   3 +-
 .../org/apache/hyracks/api/result/IResultSet.java  |   3 +
 .../hyracks/api/result/IResultSetReader.java       |   2 +
 .../hyracks/api/result/ResultDirectoryRecord.java  |  23 +-
 .../apache/hyracks/api/result/ResultJobRecord.java |  22 +-
 .../hyracks/api/util/JavaSerializationUtils.java   |  39 +-
 .../client/result/PartitionResultSetReader.java    | 112 ++++
 .../apache/hyracks/client/result/ResultSet.java    |  13 +
 .../hyracks/client/result/ResultSetReader.java     |  49 +-
 .../hyracks/control/cc/ClusterControllerIPCI.java  |   4 +-
 .../control/cc/result/IResultDirectoryService.java |   6 +-
 .../control/cc/result/ResultDirectoryService.java  |  24 +-
 .../control/cc/work/GetResultMetadataWork.java     |   3 -
 .../work/RegisterResultPartitionLocationWork.java  |   7 +-
 .../ReportResultPartitionWriteCompletionWork.java  |   7 +-
 .../control/common/base/IClusterController.java    |   5 +-
 .../hyracks/control/common/ipc/CCNCFunctions.java  |  19 +-
 .../common/ipc/ClusterControllerRemoteProxy.java   |  10 +-
 .../control/nc/result/ResultPartitionManager.java  |   9 +-
 .../control/nc/result/ResultPartitionWriter.java   |   9 +-
 .../hyracks/control/nc/result/ResultState.java     |   9 +-
 .../std/util/ByteArrayAccessibleOutputStream.java  |  18 +
 .../hyracks/hyracks-dataflow-std/pom.xml           |   1 -
 .../std/result/ResultWriterOperatorDescriptor.java |   2 +
 .../apache/hyracks/http/server/utils/HttpUtil.java |   6 +-
 .../dataflow/IndexDropOperatorNodePushable.java    |   2 +-
 .../storage/am/lsm/common/impls/LSMHarness.java    |  31 +-
 .../apache/hyracks/util/ThrowingIOFunction.java    |  20 +-
 228 files changed, 4835 insertions(+), 1558 deletions(-)
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryResultApiServlet.java
 copy hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java => asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryStatusApiServlet.java (53%)
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoRequestMessage.java
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoResponseMessage.java
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultPartitionRequestMessage.java
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultRequestMessage.java
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultResponseMessage.java
 copy asterixdb/asterix-app/src/{test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp => main/java/org/apache/asterix/app/result/fields/CreatedAtPrinter.java} (52%)
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/PartitionInfoPrinter.java
 copy asterixdb/asterix-app/src/{test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp => main/java/org/apache/asterix/app/result/fields/ResultCountPrinter.java} (55%)
 create mode 100644 asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/AsyncRequestsAPIUtil.java
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-compilation-failed/async-compilation-failed.1.async.sqlpp (98%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-exhausted-result/async-exhausted-result.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-running/async-running.1.async.sqlpp => async-deferred-improved/async-exhausted-result/async-exhausted-result.2.pollget.http} (92%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-exhausted-result/async-exhausted-result.3.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-exhausted-result/async-exhausted-result.4.delete.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-exhausted-result/async-exhausted-result.5.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-failed/async-failed.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-failed/async-failed.2.pollget.uri => async-deferred-improved/async-failed/async-failed.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-json/async-json.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-json/async-json.2.pollget.uri => async-deferred-improved/async-json/async-json.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-json/async-json.3.get.uri => async-deferred-improved/async-json/async-json.3.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-repeated/async-repeated.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.2.pollget.uri => async-deferred-improved/async-repeated/async-repeated.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-repeated/async-repeated.3.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-repeated/async-repeated.4.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-repeated/async-repeated.5.query.sqlpp (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async-running/async-running.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-running/async-running.2.pollget.uri => async-deferred-improved/async-running/async-running.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.2.pollget.uri => async-deferred-improved/async-running/async-running.3.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async-running/async-running.4.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async => async-deferred-improved/async-timeout}/async.1.async.sqlpp (93%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-running/async-running.2.pollget.uri => async-deferred-improved/async-timeout/async.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred => async-deferred-improved}/async/async.1.async.sqlpp (99%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-json/async-json.2.pollget.uri => async-deferred-improved/async/async.2.pollget.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/async/async.3.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async/async.1.async.sqlpp => async-deferred-improved/deferred/deferred.1.deferred.sqlpp} (94%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-exhausted-result/async-exhausted-result.3.get.uri => async-deferred-improved/deferred/deferred.2.get.http} (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{parquet-simple/parquet-simple.01.ddl.sqlpp => parquet-null1/parquet-null1.01.ddl.sqlpp} (95%)
 rename asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp => parquet-null1/parquet-null1.02.update.sqlpp} (80%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp => parquet-null1/parquet-null1.03.ddl.sqlpp} (75%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-running/async-running.1.async.sqlpp => copy-to/parquet-null1/parquet-null1.04.query.sqlpp} (92%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{parquet-simple/parquet-simple.01.ddl.sqlpp => parquet-null2/parquet-null2.01.ddl.sqlpp} (94%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{parquet-simple/parquet-simple.02.update.sqlpp => parquet-null2/parquet-null2.02.update.sqlpp} (78%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/{negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp => parquet-null2/parquet-null2.03.ddl.sqlpp} (75%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/{async-deferred/async-running/async-running.1.async.sqlpp => copy-to/parquet-null2/parquet-null2.04.query.sqlpp} (92%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-compilation-failed/async-compilation-failed.1.regex (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-exhausted-result/async-exhausted-result.1.ignore (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.ignore
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-failed/async-failed.1.ignore (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-failed/async-failed.2.regex (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-json/async-json.1.ignore (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.2.regexjson
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-json/async-json.3.json (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-repeated/async-repeated.1.ignore (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.2.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.3.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.4.regexjson
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-repeated/async-repeated.5.json (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-running/async-running.1.ignore (100%)
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async-running/async-running.2.json (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.3.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.4.regexjson
 copy asterixdb/asterix-app/{data/csv/empty.csv => src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.1.ignore} (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.2.regex
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/async/async.1.ignore (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.2.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.3.regexjson
 copy asterixdb/asterix-app/src/test/resources/runtimets/results/{async-deferred => async-deferred-improved}/deferred/deferred.1.ignore (100%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.2.regexjson
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null1/parquet-null1.04.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null2/parquet-null2.04.adm
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Datasets.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntSortedBitSet.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntUtil.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/MathUtil.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortSortedBitSet.java
 create mode 100644 asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortUtil.java
 create mode 100644 asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/MathUtilTest.java
 create mode 100644 asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/ShortSortedBitSetTest.java
 create mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsConstants.java
 create mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsUtils.java
 delete mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
 create mode 100644 hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/ReplacementsAwareJavaSerializationProvider.java
 create mode 100644 hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java
 copy asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java => hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIOFunction.java (60%)


(asterixdb) 17/25: [NO ISSUE][*DB][COMMON] Fix async checkpoint failure if only metadata is flushed

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit b67dac42368944d11f0224d51cf20585bc596cfa
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Oct 16 15:39:47 2025 -0400

    [NO ISSUE][*DB][COMMON] Fix async checkpoint failure if only metadata is flushed
    
    e.g.
    java.util.NoSuchElementException
      at o.a.a.c.utils.IntSortedBitSet$1.nextInt(IntSortedBitSet.java:219)
      at o.a.a.c.utils.IntUtil.appendCompact(IntUtil.java:79)
      at o.a.a.c.utils.Partitions.toString(Partitions.java:139)
      at java.lang.String.valueOf(String.java:4465)
      at o.a.a.c.context.DatasetLifecycleManager.asyncFlushMatchingIndexes(DatasetLifecycleManager.java:526)
    
    Ext-ref: MB-68946
    
    Change-Id: I081c80a54a749e15a1b3e10c3a6391835679fff6
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20491
    Reviewed-by: Michael Blow <mb...@apache.org>
    Tested-by: Michael Blow <mb...@apache.org>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
 .../src/main/java/org/apache/asterix/common/utils/Partitions.java     | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
index cc93b916cc..e939ce13b7 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -136,7 +136,9 @@ public class Partitions implements Serializable {
                 builder.append(',');
             }
         }
-        IntUtil.appendCompact(iter, builder, MINUS_ONE);
+        if (iter.hasNext()) {
+            IntUtil.appendCompact(iter, builder, MINUS_ONE);
+        }
         builder.append(']');
         return builder.toString();
     }


(asterixdb) 07/25: [NO ISSUE]: Refactor AWS common properties

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit b9d29911b46f2c17420a7d6438a9946133f89501
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Sun Oct 5 15:23:01 2025 +0300

    [NO ISSUE]: Refactor AWS common properties
    
    In an anticipation of incoming other AWS clients,
    factor out the common AWS properties to a different
    class outside of S3-specific classes.
    
    Ext-ref: MB-68835
    Change-Id: I4d2a50d361ad949d4d3aeee3208bbd8dbb3e83a2
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20463
    Tested-by: Hussain Towaileb <hu...@gmail.com>
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../external/ExternalCredentialsCacheUpdater.java  |   4 +-
 .../org/apache/asterix/utils/RedactionUtil.java    |   2 +-
 .../cloud/clients/aws/s3/S3ClientConfig.java       |   4 +-
 .../cloud/writer/S3ExternalFileWriterFactory.java  |   4 +-
 .../input/record/reader/aws/AwsS3InputStream.java  |   9 +-
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  |   2 +-
 .../aws/parquet/AwsS3ParquetReaderFactory.java     |   2 +-
 .../asterix/external/util/ExternalDataUtils.java   |  11 +-
 .../asterix/external/util/aws/AwsConstants.java    |  42 ++
 .../apache/asterix/external/util/aws/AwsUtils.java | 286 ++++++++++
 .../asterix/external/util/aws/s3/S3AuthUtils.java  | 577 ---------------------
 .../asterix/external/util/aws/s3/S3Constants.java  |  16 -
 .../asterix/external/util/aws/s3/S3Utils.java      | 317 ++++++++++-
 .../input/record/reader/awss3/AwsS3Test.java       |   4 +-
 14 files changed, 657 insertions(+), 623 deletions(-)

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
index 10f536bb71..a9541260c9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/ExternalCredentialsCacheUpdater.java
@@ -41,7 +41,7 @@ import org.apache.asterix.common.external.IExternalCredentialsCacheUpdater;
 import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.messaging.api.MessageFuture;
 import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
+import org.apache.asterix.external.util.aws.AwsUtils;
 import org.apache.asterix.messaging.CCMessageBroker;
 import org.apache.asterix.messaging.NCMessageBroker;
 import org.apache.hyracks.api.application.INCServiceContext;
@@ -94,7 +94,7 @@ public class ExternalCredentialsCacheUpdater implements IExternalCredentialsCach
             validateClusterState();
             try {
                 LOGGER.info("attempting to update AWS credentials for {}", key);
-                AwsCredentialsProvider newCredentials = S3AuthUtils.assumeRoleAndGetCredentials(configuration);
+                AwsCredentialsProvider newCredentials = AwsUtils.assumeRoleAndGetCredentials(configuration);
                 LOGGER.info("updated AWS credentials successfully for {}", key);
                 credentials = (AwsSessionCredentials) newCredentials.resolveCredentials();
                 appCtx.getExternalCredentialsCache().put(key, credentials);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
index d8b47b88b5..8ef2882fcb 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
@@ -20,7 +20,7 @@ package org.apache.asterix.utils;
 
 import static java.util.regex.Pattern.CASE_INSENSITIVE;
 import static java.util.regex.Pattern.DOTALL;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SECRET_ACCESS_KEY_FIELD_NAME;
 
 import java.util.regex.Pattern;
 
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index 025af0809f..f05a0e5820 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -22,7 +22,7 @@ import java.util.Map;
 import java.util.Objects;
 
 import org.apache.asterix.common.config.CloudProperties;
-import org.apache.asterix.external.util.aws.s3.S3Constants;
+import org.apache.asterix.external.util.aws.AwsConstants;
 
 import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
@@ -90,7 +90,7 @@ public final class S3ClientConfig {
 
     public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) {
         // Used to determine local vs. actual S3
-        String endPoint = configuration.getOrDefault(S3Constants.SERVICE_END_POINT_FIELD_NAME, "");
+        String endPoint = configuration.getOrDefault(AwsConstants.SERVICE_END_POINT_FIELD_NAME, "");
         // Disabled
         long profilerLogInterval = 0;
 
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index b3747468cf..2d618149c1 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -27,8 +27,8 @@ import org.apache.asterix.cloud.clients.aws.s3.S3CloudClient;
 import org.apache.asterix.common.api.IApplicationContext;
 import org.apache.asterix.common.exceptions.CompilationException;
 import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
 import org.apache.asterix.external.util.aws.s3.S3Constants;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
 import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
 import org.apache.asterix.runtime.writer.IExternalFileWriter;
 import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
@@ -67,7 +67,7 @@ public final class S3ExternalFileWriterFactory extends AbstractCloudExternalFile
     @Override
     ICloudClient createCloudClient(IApplicationContext appCtx) throws CompilationException {
         S3ClientConfig config = S3ClientConfig.of(configuration, writeBufferSize);
-        return new S3CloudClient(config, S3AuthUtils.buildAwsS3Client(appCtx, configuration),
+        return new S3CloudClient(config, S3Utils.buildClient(appCtx, configuration),
                 ICloudGuardian.NoOpCloudGuardian.INSTANCE);
     }
 
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
index 60205ece05..bea096671c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStream.java
@@ -35,7 +35,8 @@ import org.apache.asterix.external.input.filter.embedder.IExternalFilterValueEmb
 import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStream;
 import org.apache.asterix.external.input.record.reader.stream.AvailableInputStream;
 import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
+import org.apache.asterix.external.util.aws.AwsUtils;
+import org.apache.asterix.external.util.aws.s3.S3Utils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.util.CleanupUtils;
@@ -96,7 +97,7 @@ public class AwsS3InputStream extends AbstractExternalInputStream {
                 LOGGER.debug(() -> "Key " + userData(request.key()) + " was not found in bucket {}" + request.bucket());
                 return false;
             } catch (S3Exception ex) {
-                if (S3AuthUtils.isArnAssumedRoleExpiredToken(configuration, ex.awsErrorDetails().errorCode())) {
+                if (AwsUtils.isArnAssumedRoleExpiredToken(configuration, ex.awsErrorDetails().errorCode())) {
                     LOGGER.debug(() -> "Expired AWS assume role session, will attempt to refresh the session");
                     rebuildAwsS3Client(configuration);
                     LOGGER.debug(() -> "Successfully refreshed AWS assume role session");
@@ -120,7 +121,7 @@ public class AwsS3InputStream extends AbstractExternalInputStream {
     }
 
     private boolean shouldRetry(String errorCode, int currentRetry) {
-        return currentRetry < MAX_RETRIES && S3AuthUtils.isRetryableError(errorCode);
+        return currentRetry < MAX_RETRIES && S3Utils.isRetryableError(errorCode);
     }
 
     @Override
@@ -148,7 +149,7 @@ public class AwsS3InputStream extends AbstractExternalInputStream {
 
     private S3Client buildAwsS3Client(Map<String, String> configuration) throws HyracksDataException {
         try {
-            return S3AuthUtils.buildAwsS3Client(ncAppCtx, configuration);
+            return S3Utils.buildClient(ncAppCtx, configuration);
         } catch (CompilationException ex) {
             throw HyracksDataException.create(ex);
         }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
index 7d678a9e46..51373a1d94 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -18,7 +18,7 @@
  */
 package org.apache.asterix.external.input.record.reader.aws.delta;
 
-import static org.apache.asterix.external.util.aws.s3.S3AuthUtils.configureAwsS3HdfsJobConf;
+import static org.apache.asterix.external.util.aws.s3.S3Utils.configureAwsS3HdfsJobConf;
 
 import java.util.Collections;
 import java.util.List;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
index f5dc7a2eb6..833cba541e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/parquet/AwsS3ParquetReaderFactory.java
@@ -18,7 +18,7 @@
  */
 package org.apache.asterix.external.input.record.reader.aws.parquet;
 
-import static org.apache.asterix.external.util.aws.s3.S3AuthUtils.configureAwsS3HdfsJobConf;
+import static org.apache.asterix.external.util.aws.s3.S3Utils.configureAwsS3HdfsJobConf;
 import static org.apache.asterix.external.util.aws.s3.S3Utils.listS3Objects;
 import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 9926e09e4c..ea7b9c5a68 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -32,7 +32,7 @@ import static org.apache.asterix.external.util.ExternalDataConstants.KEY_INCLUDE
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PATH;
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_END;
 import static org.apache.asterix.external.util.ExternalDataConstants.KEY_RECORD_START;
-import static org.apache.asterix.external.util.aws.s3.S3AuthUtils.configureAwsS3HdfsJobConf;
+import static org.apache.asterix.external.util.aws.s3.S3Utils.configureAwsS3HdfsJobConf;
 import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.validateAzureBlobProperties;
 import static org.apache.asterix.external.util.azure.blob_storage.AzureUtils.validateAzureDataLakeProperties;
 import static org.apache.asterix.external.util.google.gcs.GCSAuthUtils.configureHdfsJobConf;
@@ -81,7 +81,7 @@ import org.apache.asterix.external.input.record.reader.abstracts.AbstractExterna
 import org.apache.asterix.external.library.JavaLibrary;
 import org.apache.asterix.external.library.msgpack.MessagePackUtils;
 import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
-import org.apache.asterix.external.util.aws.s3.S3AuthUtils;
+import org.apache.asterix.external.util.aws.AwsConstants;
 import org.apache.asterix.external.util.aws.s3.S3Constants;
 import org.apache.asterix.external.util.aws.s3.S3Utils;
 import org.apache.asterix.external.util.azure.blob_storage.AzureConstants;
@@ -603,8 +603,9 @@ public class ExternalDataUtils {
         // If the table is in S3
         if (configuration.get(ExternalDataConstants.KEY_READER).equals(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3)) {
 
-            conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
-            conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY, configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
+            conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(AwsConstants.ACCESS_KEY_ID_FIELD_NAME));
+            conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY,
+                    configuration.get(AwsConstants.SECRET_ACCESS_KEY_FIELD_NAME));
             tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
                     + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
                     + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
@@ -728,7 +729,7 @@ public class ExternalDataUtils {
 
         switch (type) {
             case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
-                S3AuthUtils.validateProperties(appCtx, configuration, srcLoc, collector);
+                S3Utils.validateProperties(appCtx, configuration, srcLoc, collector);
                 break;
             case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
                 validateAzureBlobProperties(configuration, srcLoc, collector, appCtx);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsConstants.java
new file mode 100644
index 0000000000..7868bdf2ca
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsConstants.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.aws;
+
+public class AwsConstants {
+    private AwsConstants() {
+        throw new AssertionError("do not instantiate");
+    }
+
+    // Authentication specific parameters
+    public static final String REGION_FIELD_NAME = "region";
+    public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
+    public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
+    public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
+    public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
+    public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
+    public static final String ROLE_ARN_FIELD_NAME = "roleArn";
+    public static final String EXTERNAL_ID_FIELD_NAME = "externalId";
+    public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
+
+    // AWS specific error codes
+    public static final String ERROR_INTERNAL_ERROR = "InternalError";
+    public static final String ERROR_SLOW_DOWN = "SlowDown";
+    public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
+    public static final String ERROR_EXPIRED_TOKEN = "ExpiredToken";
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsUtils.java
new file mode 100644
index 0000000000..a648853a46
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/AwsUtils.java
@@ -0,0 +1,286 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.util.aws;
+
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
+import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
+import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
+import static org.apache.asterix.external.util.aws.AwsConstants.ACCESS_KEY_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.CROSS_REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_EXPIRED_TOKEN;
+import static org.apache.asterix.external.util.aws.AwsConstants.EXTERNAL_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.INSTANCE_PROFILE_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.ROLE_ARN_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SECRET_ACCESS_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SESSION_TOKEN_FIELD_NAME;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.UUID;
+
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.external.IExternalCredentialsCache;
+import org.apache.asterix.common.external.IExternalCredentialsCacheUpdater;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.sts.StsClient;
+import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
+import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
+import software.amazon.awssdk.services.sts.model.Credentials;
+
+public class AwsUtils {
+
+    public enum AuthenticationType {
+        ANONYMOUS,
+        ARN_ASSUME_ROLE,
+        INSTANCE_PROFILE,
+        ACCESS_KEYS,
+        BAD_AUTHENTICATION
+    }
+
+    private AwsUtils() {
+        throw new AssertionError("do not instantiate");
+    }
+
+    public static boolean isArnAssumedRoleExpiredToken(Map<String, String> configuration, String errorCode) {
+        return ERROR_EXPIRED_TOKEN.equals(errorCode)
+                && getAuthenticationType(configuration) == AuthenticationType.ARN_ASSUME_ROLE;
+    }
+
+    public static AwsCredentialsProvider buildCredentialsProvider(IApplicationContext appCtx,
+            Map<String, String> configuration) throws CompilationException {
+        AuthenticationType authenticationType = getAuthenticationType(configuration);
+        switch (authenticationType) {
+            case ANONYMOUS:
+                return AnonymousCredentialsProvider.create();
+            case ARN_ASSUME_ROLE:
+                return getTrustAccountCredentials(appCtx, configuration);
+            case INSTANCE_PROFILE:
+                return getInstanceProfileCredentials(configuration);
+            case ACCESS_KEYS:
+                return getAccessKeyCredentials(configuration);
+            default:
+                // missing required creds, report correct error message
+                String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
+                if (externalId != null) {
+                    throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
+                            EXTERNAL_ID_FIELD_NAME);
+                } else {
+                    throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+                            SESSION_TOKEN_FIELD_NAME);
+                }
+        }
+    }
+
+    public static Region validateAndGetRegion(String regionId) throws CompilationException {
+        List<Region> regions = S3Client.serviceMetadata().regions();
+        Optional<Region> selectedRegion = regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
+
+        if (selectedRegion.isEmpty()) {
+            throw new CompilationException(S3_REGION_NOT_SUPPORTED, regionId);
+        }
+        return selectedRegion.get();
+    }
+
+    public static AuthenticationType getAuthenticationType(Map<String, String> configuration) {
+        String roleArn = configuration.get(ROLE_ARN_FIELD_NAME);
+        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+
+        if (noAuth(configuration)) {
+            return AuthenticationType.ANONYMOUS;
+        } else if (roleArn != null) {
+            return AuthenticationType.ARN_ASSUME_ROLE;
+        } else if (instanceProfile != null) {
+            return AuthenticationType.INSTANCE_PROFILE;
+        } else if (accessKeyId != null || secretAccessKey != null) {
+            return AuthenticationType.ACCESS_KEYS;
+        } else {
+            return AuthenticationType.BAD_AUTHENTICATION;
+        }
+    }
+
+    public static boolean validateAndGetCrossRegion(String crossRegion) throws CompilationException {
+        if (crossRegion == null) {
+            return false;
+        }
+        if (!"true".equalsIgnoreCase(crossRegion) && !"false".equalsIgnoreCase(crossRegion)) {
+            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, CROSS_REGION_FIELD_NAME, "true, false");
+        }
+        return Boolean.parseBoolean(crossRegion);
+    }
+
+    private static boolean noAuth(Map<String, String> configuration) {
+        return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
+                ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME, SESSION_TOKEN_FIELD_NAME) == null;
+    }
+
+    /**
+     * Returns the cached credentials if valid, otherwise, generates new credentials
+     *
+     * @param appCtx application context
+     * @param configuration configuration
+     * @return returns the cached credentials if valid, otherwise, generates new credentials
+     * @throws CompilationException CompilationException
+     */
+    public static AwsCredentialsProvider getTrustAccountCredentials(IApplicationContext appCtx,
+            Map<String, String> configuration) throws CompilationException {
+        IExternalCredentialsCache cache = appCtx.getExternalCredentialsCache();
+        Object credentialsObject = cache.get(configuration.get(ExternalDataConstants.KEY_ENTITY_ID));
+        if (credentialsObject != null) {
+            return () -> (AwsSessionCredentials) credentialsObject;
+        }
+        IExternalCredentialsCacheUpdater cacheUpdater = appCtx.getExternalCredentialsCacheUpdater();
+        AwsSessionCredentials credentials;
+        try {
+            credentials = (AwsSessionCredentials) cacheUpdater.generateAndCacheCredentials(configuration);
+        } catch (HyracksDataException ex) {
+            throw new CompilationException(ErrorCode.FAILED_EXTERNAL_CROSS_ACCOUNT_AUTHENTICATION, ex, ex.getMessage());
+        }
+
+        return () -> credentials;
+    }
+
+    /**
+     * Assume role using provided credentials and return the new credentials
+     *
+     * @param configuration configuration
+     * @return return credentials from the assume role
+     * @throws CompilationException CompilationException
+     */
+    public static AwsCredentialsProvider assumeRoleAndGetCredentials(Map<String, String> configuration)
+            throws CompilationException {
+        String regionId = configuration.get(REGION_FIELD_NAME);
+        String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
+        String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
+        Region region = validateAndGetRegion(regionId);
+
+        AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
+        builder.roleArn(arnRole);
+        builder.roleSessionName(UUID.randomUUID().toString());
+        builder.durationSeconds(900); // TODO(htowaileb): configurable? Can be 900 to 43200 (15 mins to 12 hours)
+        if (externalId != null) {
+            builder.externalId(externalId);
+        }
+        AssumeRoleRequest request = builder.build();
+        AwsCredentialsProvider credentialsProvider = getCredentialsToAssumeRole(configuration);
+
+        // assume the role from the provided arn
+        try (StsClient stsClient =
+                StsClient.builder().region(region).credentialsProvider(credentialsProvider).build()) {
+            AssumeRoleResponse response = stsClient.assumeRole(request);
+            Credentials credentials = response.credentials();
+            return StaticCredentialsProvider.create(AwsSessionCredentials.create(credentials.accessKeyId(),
+                    credentials.secretAccessKey(), credentials.sessionToken()));
+        } catch (SdkException ex) {
+            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
+        }
+    }
+
+    private static AwsCredentialsProvider getCredentialsToAssumeRole(Map<String, String> configuration)
+            throws CompilationException {
+        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+        if (instanceProfile != null) {
+            return getInstanceProfileCredentials(configuration);
+        } else if (accessKeyId != null || secretAccessKey != null) {
+            return getAccessKeyCredentials(configuration);
+        } else {
+            throw new CompilationException(ErrorCode.NO_AWS_VALID_PARAMS_FOUND_FOR_CROSS_ACCOUNT_TRUST_AUTHENTICATION);
+        }
+    }
+
+    private static AwsCredentialsProvider getInstanceProfileCredentials(Map<String, String> configuration)
+            throws CompilationException {
+        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
+
+        // only "true" value is allowed
+        if (!"true".equalsIgnoreCase(instanceProfile)) {
+            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, INSTANCE_PROFILE_FIELD_NAME, "true");
+        }
+
+        String notAllowed = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
+                SESSION_TOKEN_FIELD_NAME);
+        if (notAllowed != null) {
+            throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
+                    INSTANCE_PROFILE_FIELD_NAME);
+        }
+        return InstanceProfileCredentialsProvider.create();
+    }
+
+    private static AwsCredentialsProvider getAccessKeyCredentials(Map<String, String> configuration)
+            throws CompilationException {
+        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
+        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
+        String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
+
+        if (accessKeyId == null) {
+            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
+                    SECRET_ACCESS_KEY_FIELD_NAME);
+        }
+        if (secretAccessKey == null) {
+            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
+                    ACCESS_KEY_ID_FIELD_NAME);
+        }
+
+        // use session token if provided
+        if (sessionToken != null) {
+            return StaticCredentialsProvider
+                    .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
+        } else {
+            return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
+        }
+    }
+
+    private static String getNonNull(Map<String, String> configuration, String... fieldNames) {
+        for (String fieldName : fieldNames) {
+            if (configuration.get(fieldName) != null) {
+                return fieldName;
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Generates a random external ID to be used in cross-account role assumption.
+     *
+     * @return external id
+     */
+    public static String generateExternalId() {
+        return UUID.randomUUID().toString();
+    }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
deleted file mode 100644
index 90bb04f309..0000000000
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.util.aws.s3;
-
-import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
-import static org.apache.asterix.common.exceptions.ErrorCode.PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT;
-import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_PARAM_IS_PRESENT;
-import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
-import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
-import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
-import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableExists;
-import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
-import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.CROSS_REGION_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_EXPIRED_TOKEN;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.EXTERNAL_ID_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ACCESS_KEY_ID;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ANONYMOUS;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUMED_ROLE;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_ARN;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_EXTERNAL_ID;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_SESSION_DURATION;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_SESSION_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIALS_TO_ASSUME_ROLE_KEY;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIAL_PROVIDER_KEY;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_INSTANCE_PROFILE;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_PATH_STYLE_ACCESS;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_REGION;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_S3_CONNECTION_POOL_SIZE;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SECRET_ACCESS_KEY;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SERVICE_END_POINT;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SESSION_TOKEN;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SIMPLE;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMPORARY;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.PATH_STYLE_ADDRESSING_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ROLE_ARN_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.SESSION_TOKEN_FIELD_NAME;
-import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
-
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.UUID;
-
-import org.apache.asterix.common.api.IApplicationContext;
-import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.external.IExternalCredentialsCache;
-import org.apache.asterix.common.external.IExternalCredentialsCacheUpdater;
-import org.apache.asterix.external.util.ExternalDataConstants;
-import org.apache.asterix.external.util.ExternalDataPrefix;
-import org.apache.hadoop.fs.s3a.Constants;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
-import org.apache.hyracks.api.exceptions.SourceLocation;
-import org.apache.hyracks.api.exceptions.Warning;
-import org.apache.hyracks.api.util.CleanupUtils;
-
-import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
-import software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-import software.amazon.awssdk.core.exception.SdkException;
-import software.amazon.awssdk.regions.Region;
-import software.amazon.awssdk.services.s3.S3Client;
-import software.amazon.awssdk.services.s3.S3ClientBuilder;
-import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
-import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
-import software.amazon.awssdk.services.s3.model.S3Exception;
-import software.amazon.awssdk.services.s3.model.S3Response;
-import software.amazon.awssdk.services.sts.StsClient;
-import software.amazon.awssdk.services.sts.model.AssumeRoleRequest;
-import software.amazon.awssdk.services.sts.model.AssumeRoleResponse;
-import software.amazon.awssdk.services.sts.model.Credentials;
-
-public class S3AuthUtils {
-    enum AuthenticationType {
-        ANONYMOUS,
-        ARN_ASSUME_ROLE,
-        INSTANCE_PROFILE,
-        ACCESS_KEYS,
-        BAD_AUTHENTICATION
-    }
-
-    private S3AuthUtils() {
-        throw new AssertionError("do not instantiate");
-    }
-
-    public static boolean isRetryableError(String errorCode) {
-        return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
-    }
-
-    public static boolean isArnAssumedRoleExpiredToken(Map<String, String> configuration, String errorCode) {
-        return ERROR_EXPIRED_TOKEN.equals(errorCode)
-                && getAuthenticationType(configuration) == AuthenticationType.ARN_ASSUME_ROLE;
-    }
-
-    /**
-     * Builds the S3 client using the provided configuration
-     *
-     * @param configuration properties
-     * @return S3 client
-     * @throws CompilationException CompilationException
-     */
-    public static S3Client buildAwsS3Client(IApplicationContext appCtx, Map<String, String> configuration)
-            throws CompilationException {
-        String regionId = configuration.get(REGION_FIELD_NAME);
-        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
-
-        Region region = validateAndGetRegion(regionId);
-        boolean crossRegion = validateAndGetCrossRegion(configuration.get(CROSS_REGION_FIELD_NAME));
-        AwsCredentialsProvider credentialsProvider = buildCredentialsProvider(appCtx, configuration);
-
-        S3ClientBuilder builder = S3Client.builder();
-        builder.region(region);
-        builder.crossRegionAccessEnabled(crossRegion);
-        builder.credentialsProvider(credentialsProvider);
-
-        // Validate the service endpoint if present
-        if (serviceEndpoint != null) {
-            try {
-                URI uri = new URI(serviceEndpoint);
-                try {
-                    builder.endpointOverride(uri);
-                } catch (NullPointerException ex) {
-                    throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
-                }
-            } catch (URISyntaxException ex) {
-                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex,
-                        String.format("Invalid service endpoint %s", serviceEndpoint));
-            }
-        }
-
-        boolean pathStyleAddressing =
-                validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
-        builder.forcePathStyle(pathStyleAddressing);
-        return builder.build();
-    }
-
-    public static AwsCredentialsProvider buildCredentialsProvider(IApplicationContext appCtx,
-            Map<String, String> configuration) throws CompilationException {
-        AuthenticationType authenticationType = getAuthenticationType(configuration);
-        switch (authenticationType) {
-            case ANONYMOUS:
-                return AnonymousCredentialsProvider.create();
-            case ARN_ASSUME_ROLE:
-                return getTrustAccountCredentials(appCtx, configuration);
-            case INSTANCE_PROFILE:
-                return getInstanceProfileCredentials(configuration);
-            case ACCESS_KEYS:
-                return getAccessKeyCredentials(configuration);
-            default:
-                // missing required creds, report correct error message
-                String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
-                if (externalId != null) {
-                    throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ROLE_ARN_FIELD_NAME,
-                            EXTERNAL_ID_FIELD_NAME);
-                } else {
-                    throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
-                            SESSION_TOKEN_FIELD_NAME);
-                }
-        }
-    }
-
-    public static Region validateAndGetRegion(String regionId) throws CompilationException {
-        List<Region> regions = S3Client.serviceMetadata().regions();
-        Optional<Region> selectedRegion = regions.stream().filter(region -> region.id().equals(regionId)).findFirst();
-
-        if (selectedRegion.isEmpty()) {
-            throw new CompilationException(S3_REGION_NOT_SUPPORTED, regionId);
-        }
-        return selectedRegion.get();
-    }
-
-    private static AuthenticationType getAuthenticationType(Map<String, String> configuration) {
-        String roleArn = configuration.get(ROLE_ARN_FIELD_NAME);
-        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
-        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
-        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
-
-        if (noAuth(configuration)) {
-            return AuthenticationType.ANONYMOUS;
-        } else if (roleArn != null) {
-            return AuthenticationType.ARN_ASSUME_ROLE;
-        } else if (instanceProfile != null) {
-            return AuthenticationType.INSTANCE_PROFILE;
-        } else if (accessKeyId != null || secretAccessKey != null) {
-            return AuthenticationType.ACCESS_KEYS;
-        } else {
-            return AuthenticationType.BAD_AUTHENTICATION;
-        }
-    }
-
-    public static boolean validateAndGetPathStyleAddressing(String pathStyleAddressing, String endpoint)
-            throws CompilationException {
-        if (pathStyleAddressing == null) {
-            return endpoint != null && !endpoint.isEmpty();
-        }
-        validatePathStyleAddressing(pathStyleAddressing);
-        return Boolean.parseBoolean(pathStyleAddressing);
-    }
-
-    public static void validatePathStyleAddressing(String pathStyleAddressing) throws CompilationException {
-        if (pathStyleAddressing == null) {
-            return;
-        }
-        if (!"true".equalsIgnoreCase(pathStyleAddressing) && !"false".equalsIgnoreCase(pathStyleAddressing)) {
-            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, PATH_STYLE_ADDRESSING_FIELD_NAME,
-                    "true, false");
-        }
-    }
-
-    public static boolean validateAndGetCrossRegion(String crossRegion) throws CompilationException {
-        if (crossRegion == null) {
-            return false;
-        }
-        if (!"true".equalsIgnoreCase(crossRegion) && !"false".equalsIgnoreCase(crossRegion)) {
-            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, CROSS_REGION_FIELD_NAME, "true, false");
-        }
-        return Boolean.parseBoolean(crossRegion);
-    }
-
-    private static boolean noAuth(Map<String, String> configuration) {
-        return getNonNull(configuration, INSTANCE_PROFILE_FIELD_NAME, ROLE_ARN_FIELD_NAME, EXTERNAL_ID_FIELD_NAME,
-                ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME, SESSION_TOKEN_FIELD_NAME) == null;
-    }
-
-    /**
-     * Returns the cached credentials if valid, otherwise, generates new credentials
-     *
-     * @param appCtx application context
-     * @param configuration configuration
-     * @return returns the cached credentials if valid, otherwise, generates new credentials
-     * @throws CompilationException CompilationException
-     */
-    public static AwsCredentialsProvider getTrustAccountCredentials(IApplicationContext appCtx,
-            Map<String, String> configuration) throws CompilationException {
-        IExternalCredentialsCache cache = appCtx.getExternalCredentialsCache();
-        Object credentialsObject = cache.get(configuration.get(ExternalDataConstants.KEY_ENTITY_ID));
-        if (credentialsObject != null) {
-            return () -> (AwsSessionCredentials) credentialsObject;
-        }
-        IExternalCredentialsCacheUpdater cacheUpdater = appCtx.getExternalCredentialsCacheUpdater();
-        AwsSessionCredentials credentials;
-        try {
-            credentials = (AwsSessionCredentials) cacheUpdater.generateAndCacheCredentials(configuration);
-        } catch (HyracksDataException ex) {
-            throw new CompilationException(ErrorCode.FAILED_EXTERNAL_CROSS_ACCOUNT_AUTHENTICATION, ex, ex.getMessage());
-        }
-
-        return () -> credentials;
-    }
-
-    /**
-     * Assume role using provided credentials and return the new credentials
-     *
-     * @param configuration configuration
-     * @return return credentials from the assume role
-     * @throws CompilationException CompilationException
-     */
-    public static AwsCredentialsProvider assumeRoleAndGetCredentials(Map<String, String> configuration)
-            throws CompilationException {
-        String regionId = configuration.get(REGION_FIELD_NAME);
-        String arnRole = configuration.get(ROLE_ARN_FIELD_NAME);
-        String externalId = configuration.get(EXTERNAL_ID_FIELD_NAME);
-        Region region = validateAndGetRegion(regionId);
-
-        AssumeRoleRequest.Builder builder = AssumeRoleRequest.builder();
-        builder.roleArn(arnRole);
-        builder.roleSessionName(UUID.randomUUID().toString());
-        builder.durationSeconds(900); // TODO(htowaileb): configurable? Can be 900 to 43200 (15 mins to 12 hours)
-        if (externalId != null) {
-            builder.externalId(externalId);
-        }
-        AssumeRoleRequest request = builder.build();
-        AwsCredentialsProvider credentialsProvider = getCredentialsToAssumeRole(configuration);
-
-        // assume the role from the provided arn
-        try (StsClient stsClient =
-                StsClient.builder().region(region).credentialsProvider(credentialsProvider).build()) {
-            AssumeRoleResponse response = stsClient.assumeRole(request);
-            Credentials credentials = response.credentials();
-            return StaticCredentialsProvider.create(AwsSessionCredentials.create(credentials.accessKeyId(),
-                    credentials.secretAccessKey(), credentials.sessionToken()));
-        } catch (SdkException ex) {
-            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
-        }
-    }
-
-    private static AwsCredentialsProvider getCredentialsToAssumeRole(Map<String, String> configuration)
-            throws CompilationException {
-        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
-        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
-        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
-        if (instanceProfile != null) {
-            return getInstanceProfileCredentials(configuration);
-        } else if (accessKeyId != null || secretAccessKey != null) {
-            return getAccessKeyCredentials(configuration);
-        } else {
-            throw new CompilationException(ErrorCode.NO_AWS_VALID_PARAMS_FOUND_FOR_CROSS_ACCOUNT_TRUST_AUTHENTICATION);
-        }
-    }
-
-    private static AwsCredentialsProvider getInstanceProfileCredentials(Map<String, String> configuration)
-            throws CompilationException {
-        String instanceProfile = configuration.get(INSTANCE_PROFILE_FIELD_NAME);
-
-        // only "true" value is allowed
-        if (!"true".equalsIgnoreCase(instanceProfile)) {
-            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, INSTANCE_PROFILE_FIELD_NAME, "true");
-        }
-
-        String notAllowed = getNonNull(configuration, ACCESS_KEY_ID_FIELD_NAME, SECRET_ACCESS_KEY_FIELD_NAME,
-                SESSION_TOKEN_FIELD_NAME);
-        if (notAllowed != null) {
-            throw new CompilationException(PARAM_NOT_ALLOWED_IF_PARAM_IS_PRESENT, notAllowed,
-                    INSTANCE_PROFILE_FIELD_NAME);
-        }
-        return InstanceProfileCredentialsProvider.create();
-    }
-
-    private static AwsCredentialsProvider getAccessKeyCredentials(Map<String, String> configuration)
-            throws CompilationException {
-        String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
-        String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
-        String sessionToken = configuration.get(SESSION_TOKEN_FIELD_NAME);
-
-        if (accessKeyId == null) {
-            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, ACCESS_KEY_ID_FIELD_NAME,
-                    SECRET_ACCESS_KEY_FIELD_NAME);
-        }
-        if (secretAccessKey == null) {
-            throw new CompilationException(REQUIRED_PARAM_IF_PARAM_IS_PRESENT, SECRET_ACCESS_KEY_FIELD_NAME,
-                    ACCESS_KEY_ID_FIELD_NAME);
-        }
-
-        // use session token if provided
-        if (sessionToken != null) {
-            return StaticCredentialsProvider
-                    .create(AwsSessionCredentials.create(accessKeyId, secretAccessKey, sessionToken));
-        } else {
-            return StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKeyId, secretAccessKey));
-        }
-    }
-
-    private static String getNonNull(Map<String, String> configuration, String... fieldNames) {
-        for (String fieldName : fieldNames) {
-            if (configuration.get(fieldName) != null) {
-                return fieldName;
-            }
-        }
-        return null;
-    }
-
-    public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx, JobConf conf,
-            Map<String, String> configuration) throws CompilationException {
-        configureAwsS3HdfsJobConf(appCtx, conf, configuration, 0);
-    }
-
-    /**
-     * Builds the S3 client using the provided configuration
-     *
-     * @param appCtx application context
-     * @param configuration      properties
-     * @param numberOfPartitions number of partitions in the cluster
-     */
-    public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx, JobConf jobConf,
-            Map<String, String> configuration, int numberOfPartitions) throws CompilationException {
-        setHadoopCredentials(appCtx, jobConf, configuration);
-        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
-        Region region = validateAndGetRegion(configuration.get(REGION_FIELD_NAME));
-        jobConf.set(HADOOP_REGION, region.toString());
-        if (serviceEndpoint != null) {
-            // Validation of the URL should be done at hadoop-aws level
-            jobConf.set(HADOOP_SERVICE_END_POINT, serviceEndpoint);
-        } else {
-            //Region is ignored and buckets could be found by the central endpoint
-            jobConf.set(HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
-        }
-
-        boolean pathStyleAddressing =
-                validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
-        if (pathStyleAddressing) {
-            jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
-        } else {
-            jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.FALSE);
-        }
-
-        /*
-         * Set the size of S3 connection pool to be the number of partitions
-         */
-        if (numberOfPartitions != 0) {
-            jobConf.set(HADOOP_S3_CONNECTION_POOL_SIZE, String.valueOf(numberOfPartitions));
-        }
-    }
-
-    /**
-     * Sets the credentials provider type and the credentials to hadoop based on the provided configuration
-     *
-     * @param jobConf hadoop job config
-     * @param configuration external details configuration
-     */
-    private static void setHadoopCredentials(IApplicationContext appCtx, JobConf jobConf,
-            Map<String, String> configuration) throws CompilationException {
-        AuthenticationType authenticationType = getAuthenticationType(configuration);
-        switch (authenticationType) {
-            case ANONYMOUS:
-                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ANONYMOUS);
-                break;
-            case ARN_ASSUME_ROLE:
-                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ASSUMED_ROLE);
-                jobConf.set(HADOOP_ASSUME_ROLE_ARN, configuration.get(ROLE_ARN_FIELD_NAME));
-                jobConf.set(HADOOP_ASSUME_ROLE_EXTERNAL_ID, configuration.get(EXTERNAL_ID_FIELD_NAME));
-                jobConf.set(HADOOP_ASSUME_ROLE_SESSION_NAME, "parquet-" + UUID.randomUUID());
-
-                // hadoop accepts time 15m to 1h, we will base it on the provided configuration
-                int durationInSeconds = appCtx.getExternalProperties().getAwsAssumeRoleDuration();
-                String hadoopDuration = getHadoopDuration(durationInSeconds);
-                jobConf.set(HADOOP_ASSUME_ROLE_SESSION_DURATION, hadoopDuration);
-
-                // TODO: this assumes basic keys always, also support if we use InstanceProfile to assume a role
-                jobConf.set(HADOOP_CREDENTIALS_TO_ASSUME_ROLE_KEY, HADOOP_SIMPLE);
-                jobConf.set(HADOOP_ACCESS_KEY_ID, configuration.get(ACCESS_KEY_ID_FIELD_NAME));
-                jobConf.set(HADOOP_SECRET_ACCESS_KEY, configuration.get(SECRET_ACCESS_KEY_FIELD_NAME));
-                break;
-            case INSTANCE_PROFILE:
-                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_INSTANCE_PROFILE);
-                break;
-            case ACCESS_KEYS:
-                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_SIMPLE);
-                jobConf.set(HADOOP_ACCESS_KEY_ID, configuration.get(ACCESS_KEY_ID_FIELD_NAME));
-                jobConf.set(HADOOP_SECRET_ACCESS_KEY, configuration.get(SECRET_ACCESS_KEY_FIELD_NAME));
-                if (configuration.get(SESSION_TOKEN_FIELD_NAME) != null) {
-                    jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_TEMPORARY);
-                    jobConf.set(HADOOP_SESSION_TOKEN, configuration.get(SESSION_TOKEN_FIELD_NAME));
-                }
-                break;
-            case BAD_AUTHENTICATION:
-                throw new CompilationException(ErrorCode.NO_VALID_AUTHENTICATION_PARAMS_PROVIDED);
-        }
-    }
-
-    /**
-     * Hadoop accepts duration values from 15m to 1h (in this format). We will base this on the configured
-     * duration in seconds. If the time exceeds 1 hour, we will return 1h
-     *
-     * @param seconds configured duration in seconds
-     * @return hadoop updated duration
-     */
-    private static String getHadoopDuration(int seconds) {
-        // constants for time thresholds
-        final int FIFTEEN_MINUTES_IN_SECONDS = 15 * 60;
-        final int ONE_HOUR_IN_SECONDS = 60 * 60;
-
-        // Adjust seconds to fit within bounds
-        if (seconds < FIFTEEN_MINUTES_IN_SECONDS) {
-            seconds = FIFTEEN_MINUTES_IN_SECONDS;
-        } else if (seconds > ONE_HOUR_IN_SECONDS) {
-            seconds = ONE_HOUR_IN_SECONDS;
-        }
-
-        // Convert seconds to minutes
-        int minutes = seconds / 60;
-
-        // Format the result
-        if (minutes == 60) {
-            return "1h";
-        } else {
-            return minutes + "m";
-        }
-    }
-
-    /**
-     * Validate external dataset properties
-     *
-     * @param configuration properties
-     * @throws CompilationException Compilation exception
-     */
-    public static void validateProperties(IApplicationContext appCtx, Map<String, String> configuration,
-            SourceLocation srcLoc, IWarningCollector collector) throws CompilationException {
-        if (isDeltaTable(configuration)) {
-            validateDeltaTableProperties(configuration);
-        }
-        // check if the format property is present
-        else if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
-            throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
-        }
-
-        validateIncludeExclude(configuration);
-        try {
-            // TODO(htowaileb): maybe something better, this will check to ensure type is supported before creation
-            new ExternalDataPrefix(configuration);
-        } catch (AlgebricksException ex) {
-            throw new CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex);
-        }
-
-        // Check if the bucket is present
-        S3Client s3Client = buildAwsS3Client(appCtx, configuration);
-        S3Response response;
-        boolean useOldApi = false;
-        String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-        String prefix = getPrefix(configuration);
-
-        try {
-            response = S3Utils.isBucketEmpty(s3Client, container, prefix, false);
-        } catch (S3Exception ex) {
-            // Method not implemented, try falling back to old API
-            try {
-                // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
-                if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
-                    useOldApi = true;
-                    response = S3Utils.isBucketEmpty(s3Client, container, prefix, true);
-                } else {
-                    throw ex;
-                }
-            } catch (SdkException ex2) {
-                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex2, getMessageOrToString(ex));
-            }
-        } catch (SdkException ex) {
-            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
-        } finally {
-            if (s3Client != null) {
-                CleanupUtils.close(s3Client, null);
-            }
-        }
-
-        boolean isEmpty = useOldApi ? ((ListObjectsResponse) response).contents().isEmpty()
-                : ((ListObjectsV2Response) response).contents().isEmpty();
-        if (isEmpty && collector.shouldWarn()) {
-            Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
-            collector.warn(warning);
-        }
-
-        // Returns 200 only in case the bucket exists, otherwise, throws an exception. However, to
-        // ensure coverage, check if the result is successful as well and not only catch exceptions
-        if (!response.sdkHttpResponse().isSuccessful()) {
-            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
-        }
-        if (isDeltaTable(configuration)) {
-            try {
-                validateDeltaTableExists(appCtx, configuration);
-            } catch (AlgebricksException e) {
-                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, e);
-            }
-        }
-    }
-}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
index 102c378cca..3cfef1e996 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Constants.java
@@ -26,23 +26,7 @@ public class S3Constants {
     // Key max length
     public static final int MAX_KEY_LENGTH_IN_BYTES = 1024;
 
-    // Authentication specific parameters
-    public static final String REGION_FIELD_NAME = "region";
     public static final String PATH_STYLE_ADDRESSING_FIELD_NAME = "pathStyleAddressing";
-    public static final String CROSS_REGION_FIELD_NAME = "crossRegion";
-    public static final String INSTANCE_PROFILE_FIELD_NAME = "instanceProfile";
-    public static final String ACCESS_KEY_ID_FIELD_NAME = "accessKeyId";
-    public static final String SECRET_ACCESS_KEY_FIELD_NAME = "secretAccessKey";
-    public static final String SESSION_TOKEN_FIELD_NAME = "sessionToken";
-    public static final String ROLE_ARN_FIELD_NAME = "roleArn";
-    public static final String EXTERNAL_ID_FIELD_NAME = "externalId";
-    public static final String SERVICE_END_POINT_FIELD_NAME = "serviceEndpoint";
-
-    // AWS S3 specific error codes
-    public static final String ERROR_INTERNAL_ERROR = "InternalError";
-    public static final String ERROR_SLOW_DOWN = "SlowDown";
-    public static final String ERROR_METHOD_NOT_IMPLEMENTED = "NotImplemented";
-    public static final String ERROR_EXPIRED_TOKEN = "ExpiredToken";
 
     /*
      * Hadoop-AWS
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 7783456871..e20e39f2bf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -18,10 +18,50 @@
  */
 package org.apache.asterix.external.util.aws.s3;
 
+import static org.apache.asterix.common.exceptions.ErrorCode.INVALID_PARAM_VALUE_ALLOWED_VALUE;
 import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableExists;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
+import static org.apache.asterix.external.util.aws.AwsConstants.ACCESS_KEY_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.CROSS_REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.AwsConstants.EXTERNAL_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.ROLE_ARN_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SECRET_ACCESS_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SERVICE_END_POINT_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsConstants.SESSION_TOKEN_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.AwsUtils.buildCredentialsProvider;
+import static org.apache.asterix.external.util.aws.AwsUtils.getAuthenticationType;
+import static org.apache.asterix.external.util.aws.AwsUtils.validateAndGetCrossRegion;
+import static org.apache.asterix.external.util.aws.AwsUtils.validateAndGetRegion;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ACCESS_KEY_ID;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ANONYMOUS;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUMED_ROLE;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_ARN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_EXTERNAL_ID;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_SESSION_DURATION;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ASSUME_ROLE_SESSION_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIALS_TO_ASSUME_ROLE_KEY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIAL_PROVIDER_KEY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_INSTANCE_PROFILE;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_PATH_STYLE_ACCESS;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_REGION;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_S3_CONNECTION_POOL_SIZE;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SECRET_ACCESS_KEY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SERVICE_END_POINT;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SESSION_TOKEN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SIMPLE;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMPORARY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.PATH_STYLE_ADDRESSING_FIELD_NAME;
 import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -38,13 +78,21 @@ import org.apache.asterix.external.input.record.reader.abstracts.AbstractExterna
 import org.apache.asterix.external.util.ExternalDataConstants;
 import org.apache.asterix.external.util.ExternalDataPrefix;
 import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.aws.AwsUtils;
+import org.apache.hadoop.fs.s3a.Constants;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.SourceLocation;
 import org.apache.hyracks.api.exceptions.Warning;
 import org.apache.hyracks.api.util.CleanupUtils;
 
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
 import software.amazon.awssdk.core.exception.SdkException;
+import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3ClientBuilder;
 import software.amazon.awssdk.services.s3.model.CommonPrefix;
 import software.amazon.awssdk.services.s3.model.ListObjectsRequest;
 import software.amazon.awssdk.services.s3.model.ListObjectsResponse;
@@ -60,6 +108,241 @@ public class S3Utils {
         throw new AssertionError("do not instantiate");
     }
 
+    /**
+     * Builds the S3 client using the provided configuration
+     *
+     * @param configuration properties
+     * @return S3 client
+     * @throws CompilationException CompilationException
+     */
+    public static S3Client buildClient(IApplicationContext appCtx, Map<String, String> configuration)
+            throws CompilationException {
+        String regionId = configuration.get(REGION_FIELD_NAME);
+        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+
+        Region region = validateAndGetRegion(regionId);
+        boolean crossRegion = validateAndGetCrossRegion(configuration.get(CROSS_REGION_FIELD_NAME));
+        AwsCredentialsProvider credentialsProvider = buildCredentialsProvider(appCtx, configuration);
+
+        S3ClientBuilder builder = S3Client.builder();
+        builder.region(region);
+        builder.crossRegionAccessEnabled(crossRegion);
+        builder.credentialsProvider(credentialsProvider);
+
+        // Validate the service endpoint if present
+        if (serviceEndpoint != null) {
+            try {
+                URI uri = new URI(serviceEndpoint);
+                try {
+                    builder.endpointOverride(uri);
+                } catch (NullPointerException ex) {
+                    throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
+                }
+            } catch (URISyntaxException ex) {
+                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex,
+                        String.format("Invalid service endpoint %s", serviceEndpoint));
+            }
+        }
+
+        boolean pathStyleAddressing =
+                validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
+        builder.forcePathStyle(pathStyleAddressing);
+        return builder.build();
+    }
+
+    public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx, JobConf conf,
+            Map<String, String> configuration) throws CompilationException {
+        configureAwsS3HdfsJobConf(appCtx, conf, configuration, 0);
+    }
+
+    /**
+     * Builds the S3 client using the provided configuration
+     *
+     * @param appCtx application context
+     * @param configuration      properties
+     * @param numberOfPartitions number of partitions in the cluster
+     */
+    public static void configureAwsS3HdfsJobConf(IApplicationContext appCtx, JobConf jobConf,
+            Map<String, String> configuration, int numberOfPartitions) throws CompilationException {
+        setHadoopCredentials(appCtx, jobConf, configuration);
+        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+        Region region = validateAndGetRegion(configuration.get(REGION_FIELD_NAME));
+        jobConf.set(HADOOP_REGION, region.toString());
+        if (serviceEndpoint != null) {
+            // Validation of the URL should be done at hadoop-aws level
+            jobConf.set(HADOOP_SERVICE_END_POINT, serviceEndpoint);
+        } else {
+            //Region is ignored and buckets could be found by the central endpoint
+            jobConf.set(HADOOP_SERVICE_END_POINT, Constants.CENTRAL_ENDPOINT);
+        }
+
+        boolean pathStyleAddressing =
+                validateAndGetPathStyleAddressing(configuration.get(PATH_STYLE_ADDRESSING_FIELD_NAME), serviceEndpoint);
+        if (pathStyleAddressing) {
+            jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.TRUE);
+        } else {
+            jobConf.set(HADOOP_PATH_STYLE_ACCESS, ExternalDataConstants.FALSE);
+        }
+
+        /*
+         * Set the size of S3 connection pool to be the number of partitions
+         */
+        if (numberOfPartitions != 0) {
+            jobConf.set(HADOOP_S3_CONNECTION_POOL_SIZE, String.valueOf(numberOfPartitions));
+        }
+    }
+
+    /**
+     * Sets the credentials provider type and the credentials to hadoop based on the provided configuration
+     *
+     * @param jobConf hadoop job config
+     * @param configuration external details configuration
+     */
+    private static void setHadoopCredentials(IApplicationContext appCtx, JobConf jobConf,
+            Map<String, String> configuration) throws CompilationException {
+        AwsUtils.AuthenticationType authenticationType = getAuthenticationType(configuration);
+        switch (authenticationType) {
+            case ANONYMOUS:
+                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ANONYMOUS);
+                break;
+            case ARN_ASSUME_ROLE:
+                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_ASSUMED_ROLE);
+                jobConf.set(HADOOP_ASSUME_ROLE_ARN, configuration.get(ROLE_ARN_FIELD_NAME));
+                jobConf.set(HADOOP_ASSUME_ROLE_EXTERNAL_ID, configuration.get(EXTERNAL_ID_FIELD_NAME));
+                jobConf.set(HADOOP_ASSUME_ROLE_SESSION_NAME, "parquet-" + UUID.randomUUID());
+
+                // hadoop accepts time 15m to 1h, we will base it on the provided configuration
+                int durationInSeconds = appCtx.getExternalProperties().getAwsAssumeRoleDuration();
+                String hadoopDuration = getHadoopDuration(durationInSeconds);
+                jobConf.set(HADOOP_ASSUME_ROLE_SESSION_DURATION, hadoopDuration);
+
+                // TODO: this assumes basic keys always, also support if we use InstanceProfile to assume a role
+                jobConf.set(HADOOP_CREDENTIALS_TO_ASSUME_ROLE_KEY, HADOOP_SIMPLE);
+                jobConf.set(HADOOP_ACCESS_KEY_ID, configuration.get(ACCESS_KEY_ID_FIELD_NAME));
+                jobConf.set(HADOOP_SECRET_ACCESS_KEY, configuration.get(SECRET_ACCESS_KEY_FIELD_NAME));
+                break;
+            case INSTANCE_PROFILE:
+                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_INSTANCE_PROFILE);
+                break;
+            case ACCESS_KEYS:
+                jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_SIMPLE);
+                jobConf.set(HADOOP_ACCESS_KEY_ID, configuration.get(ACCESS_KEY_ID_FIELD_NAME));
+                jobConf.set(HADOOP_SECRET_ACCESS_KEY, configuration.get(SECRET_ACCESS_KEY_FIELD_NAME));
+                if (configuration.get(SESSION_TOKEN_FIELD_NAME) != null) {
+                    jobConf.set(HADOOP_CREDENTIAL_PROVIDER_KEY, HADOOP_TEMPORARY);
+                    jobConf.set(HADOOP_SESSION_TOKEN, configuration.get(SESSION_TOKEN_FIELD_NAME));
+                }
+                break;
+            case BAD_AUTHENTICATION:
+                throw new CompilationException(ErrorCode.NO_VALID_AUTHENTICATION_PARAMS_PROVIDED);
+        }
+    }
+
+    /**
+     * Hadoop accepts duration values from 15m to 1h (in this format). We will base this on the configured
+     * duration in seconds. If the time exceeds 1 hour, we will return 1h
+     *
+     * @param seconds configured duration in seconds
+     * @return hadoop updated duration
+     */
+    private static String getHadoopDuration(int seconds) {
+        // constants for time thresholds
+        final int FIFTEEN_MINUTES_IN_SECONDS = 15 * 60;
+        final int ONE_HOUR_IN_SECONDS = 60 * 60;
+
+        // Adjust seconds to fit within bounds
+        if (seconds < FIFTEEN_MINUTES_IN_SECONDS) {
+            seconds = FIFTEEN_MINUTES_IN_SECONDS;
+        } else if (seconds > ONE_HOUR_IN_SECONDS) {
+            seconds = ONE_HOUR_IN_SECONDS;
+        }
+
+        // Convert seconds to minutes
+        int minutes = seconds / 60;
+
+        // Format the result
+        if (minutes == 60) {
+            return "1h";
+        } else {
+            return minutes + "m";
+        }
+    }
+
+    /**
+     * Validate external dataset properties
+     *
+     * @param configuration properties
+     * @throws CompilationException Compilation exception
+     */
+    public static void validateProperties(IApplicationContext appCtx, Map<String, String> configuration,
+            SourceLocation srcLoc, IWarningCollector collector) throws CompilationException {
+        if (isDeltaTable(configuration)) {
+            validateDeltaTableProperties(configuration);
+        }
+        // check if the format property is present
+        else if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
+            throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
+        }
+
+        validateIncludeExclude(configuration);
+        try {
+            // TODO(htowaileb): maybe something better, this will check to ensure type is supported before creation
+            new ExternalDataPrefix(configuration);
+        } catch (AlgebricksException ex) {
+            throw new CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex);
+        }
+
+        // Check if the bucket is present
+        S3Client s3Client = buildClient(appCtx, configuration);
+        S3Response response;
+        boolean useOldApi = false;
+        String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
+        String prefix = getPrefix(configuration);
+
+        try {
+            response = S3Utils.isBucketEmpty(s3Client, container, prefix, false);
+        } catch (S3Exception ex) {
+            // Method not implemented, try falling back to old API
+            try {
+                // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+                if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
+                    useOldApi = true;
+                    response = S3Utils.isBucketEmpty(s3Client, container, prefix, true);
+                } else {
+                    throw ex;
+                }
+            } catch (SdkException ex2) {
+                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex2, getMessageOrToString(ex));
+            }
+        } catch (SdkException ex) {
+            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
+        } finally {
+            if (s3Client != null) {
+                CleanupUtils.close(s3Client, null);
+            }
+        }
+
+        boolean isEmpty = useOldApi ? ((ListObjectsResponse) response).contents().isEmpty()
+                : ((ListObjectsV2Response) response).contents().isEmpty();
+        if (isEmpty && collector.shouldWarn()) {
+            Warning warning = Warning.of(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
+            collector.warn(warning);
+        }
+
+        // Returns 200 only in case the bucket exists, otherwise, throws an exception. However, to
+        // ensure coverage, check if the result is successful as well and not only catch exceptions
+        if (!response.sdkHttpResponse().isSuccessful()) {
+            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
+        }
+        if (isDeltaTable(configuration)) {
+            try {
+                validateDeltaTableExists(appCtx, configuration);
+            } catch (AlgebricksException e) {
+                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, e);
+            }
+        }
+    }
+
     /**
      * Checks for a single object in the specified bucket to determine if the bucket is empty or not.
      *
@@ -96,7 +379,7 @@ public class S3Utils {
         // Prepare to retrieve the objects
         List<S3Object> filesOnly;
         String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
-        S3Client s3Client = S3AuthUtils.buildAwsS3Client(appCtx, configuration);
+        S3Client s3Client = buildClient(appCtx, configuration);
         String prefix = getPrefix(configuration);
 
         try {
@@ -243,7 +526,7 @@ public class S3Utils {
             Map<String, String> configuration, String container, String prefix)
             throws CompilationException, HyracksDataException {
         // create s3 client
-        S3Client s3Client = S3AuthUtils.buildAwsS3Client(appCtx, configuration);
+        S3Client s3Client = buildClient(appCtx, configuration);
         // fetch all the s3 objects
         return listS3ObjectsOfSingleDepth(s3Client, container, prefix);
     }
@@ -300,12 +583,26 @@ public class S3Utils {
                 + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
     }
 
-    /**
-     * Generates a random external ID to be used in cross-account role assumption.
-     *
-     * @return external id
-     */
-    public static String generateExternalId() {
-        return UUID.randomUUID().toString();
+    public static boolean isRetryableError(String errorCode) {
+        return errorCode.equals(ERROR_INTERNAL_ERROR) || errorCode.equals(ERROR_SLOW_DOWN);
+    }
+
+    public static boolean validateAndGetPathStyleAddressing(String pathStyleAddressing, String endpoint)
+            throws CompilationException {
+        if (pathStyleAddressing == null) {
+            return endpoint != null && !endpoint.isEmpty();
+        }
+        validatePathStyleAddressing(pathStyleAddressing);
+        return "true".equalsIgnoreCase(pathStyleAddressing);
+    }
+
+    public static void validatePathStyleAddressing(String pathStyleAddressing) throws CompilationException {
+        if (pathStyleAddressing == null) {
+            return;
+        }
+        if (!"true".equalsIgnoreCase(pathStyleAddressing) && !"false".equalsIgnoreCase(pathStyleAddressing)) {
+            throw new CompilationException(INVALID_PARAM_VALUE_ALLOWED_VALUE, PATH_STYLE_ADDRESSING_FIELD_NAME,
+                    "true, false");
+        }
     }
 }
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
index 91afbd8279..2f8791ed41 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/awss3/AwsS3Test.java
@@ -18,8 +18,8 @@
  */
 package org.apache.asterix.external.input.record.reader.awss3;
 
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.aws.AwsConstants.ERROR_SLOW_DOWN;
 
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;


(asterixdb) 11/25: [NO ISSUE][*DB][COMMON] Reduce logging on dataset async flush batches

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 69840d4929ba1c3e774cee3fc29db984d139a062
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Oct 10 18:03:50 2025 -0400

    [NO ISSUE][*DB][COMMON] Reduce logging on dataset async flush batches
    
    Ext-ref: MB-68946
    Change-Id: I1c2aaa1dbaadf00173e4e955c15c835e5b461ed7
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20479
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
 .../common/context/DatasetLifecycleManager.java    |  19 +-
 .../org/apache/asterix/common/utils/Datasets.java  | 105 +++++++++
 .../asterix/common/utils/IntSortedBitSet.java      | 234 ++++++++++++++++++++
 .../org/apache/asterix/common/utils/IntUtil.java   | 119 +++++++++++
 .../org/apache/asterix/common/utils/MathUtil.java  |  53 +++++
 .../apache/asterix/common/utils/Partitions.java    | 110 ++++++++++
 .../asterix/common/utils/ShortSortedBitSet.java    | 235 +++++++++++++++++++++
 .../org/apache/asterix/common/utils/ShortUtil.java | 122 +++++++++++
 .../apache/asterix/test/utils/MathUtilTest.java    |  79 +++++++
 .../asterix/test/utils/ShortSortedBitSetTest.java  | 108 ++++++++++
 10 files changed, 1183 insertions(+), 1 deletion(-)

diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
index 96f72415e9..e433264b25 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/DatasetLifecycleManager.java
@@ -23,6 +23,7 @@ import static org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId.MIN_
 import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -47,6 +48,8 @@ import org.apache.asterix.common.transactions.ILogManager;
 import org.apache.asterix.common.transactions.IRecoveryManager;
 import org.apache.asterix.common.transactions.LogRecord;
 import org.apache.asterix.common.transactions.LogType;
+import org.apache.asterix.common.utils.Datasets;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.hyracks.api.application.INCServiceContext;
 import org.apache.hyracks.api.exceptions.ErrorCode;
@@ -500,12 +503,27 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC
     @Override
     public synchronized void asyncFlushMatchingIndexes(Predicate<ILSMIndex> indexPredicate)
             throws HyracksDataException {
+        // Key: PartitionsBitSet (immutable after insertion) -> datasets sharing that partition set
+        Map<Partitions, Datasets> grouped = new LinkedHashMap<>();
+
         for (DatasetResource dsr : datasets.values()) {
+            Partitions flushedPartitions;
+            flushedPartitions = new Partitions();
             for (PrimaryIndexOperationTracker opTracker : dsr.getOpTrackers()) {
                 synchronized (opTracker) {
                     asyncFlush(dsr, opTracker, indexPredicate);
+                    flushedPartitions.add(opTracker.partition);
                 }
             }
+            // only log datasets for which a flush was actually scheduled
+            if (!flushedPartitions.isEmpty()) {
+                grouped.computeIfAbsent(flushedPartitions, k -> new Datasets()).add(dsr.getDatasetID());
+                // IMPORTANT: do not modify flushedPartitions after this point as it is now part of the map key!
+            }
+        }
+
+        for (Map.Entry<Partitions, Datasets> e : grouped.entrySet()) {
+            LOGGER.info("Async flushed dataset(s) " + e.getValue() + " on partition(s) " + e.getKey());
         }
     }
 
@@ -515,7 +533,6 @@ public class DatasetLifecycleManager implements IDatasetLifecycleManager, ILifeC
         for (ILSMIndex lsmIndex : dsr.getDatasetInfo().getDatasetPartitionOpenIndexes(partition)) {
             LSMIOOperationCallback ioCallback = (LSMIOOperationCallback) lsmIndex.getIOOperationCallback();
             if (needsFlush(opTracker, lsmIndex, ioCallback) && indexPredicate.test(lsmIndex)) {
-                LOGGER.info("Async flushing {}", opTracker);
                 opTracker.setFlushOnExit(true);
                 opTracker.flushIfNeeded();
                 break;
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Datasets.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Datasets.java
new file mode 100644
index 0000000000..51c1e9d1f7
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Datasets.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.function.IntPredicate;
+
+import it.unimi.dsi.fastutil.ints.IntAVLTreeSet;
+import it.unimi.dsi.fastutil.ints.IntCollection;
+import it.unimi.dsi.fastutil.ints.IntSortedSet;
+
+/**
+ * A specialized set for storing dataset IDs.
+ */
+public class Datasets implements Serializable {
+    @Serial
+    private static final long serialVersionUID = 1L;
+    private final IntSortedSet delegate;
+
+    public Datasets() {
+        this.delegate = new IntAVLTreeSet();
+    }
+
+    public boolean add(int k) {
+        return delegate.add(k);
+    }
+
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof Datasets other)) {
+            return false;
+        }
+        return delegate.equals(other.delegate);
+    }
+
+    public boolean contains(int i) {
+        return delegate.contains(i);
+    }
+
+    public boolean addAll(IntCollection intCollection) {
+        return delegate.addAll(intCollection);
+    }
+
+    public boolean containsAll(IntCollection intCollection) {
+        return delegate.containsAll(intCollection);
+    }
+
+    public boolean removeIf(IntPredicate filter) {
+        return delegate.removeIf(filter);
+    }
+
+    public boolean removeAll(IntCollection intCollection) {
+        return delegate.removeAll(intCollection);
+    }
+
+    public boolean retainAll(IntCollection intCollection) {
+        return delegate.retainAll(intCollection);
+    }
+
+    public int[] toArray() {
+        return delegate.toIntArray();
+    }
+
+    public int size() {
+        return delegate.size();
+    }
+
+    @Override
+    public int hashCode() {
+        return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        return IntUtil.toCompactString(delegate.iterator());
+    }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntSortedBitSet.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntSortedBitSet.java
new file mode 100644
index 0000000000..8d151040f6
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntSortedBitSet.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+import it.unimi.dsi.fastutil.ints.AbstractIntSortedSet;
+import it.unimi.dsi.fastutil.ints.IntBidirectionalIterator;
+import it.unimi.dsi.fastutil.ints.IntCollection;
+import it.unimi.dsi.fastutil.ints.IntComparator;
+import it.unimi.dsi.fastutil.ints.IntComparators;
+import it.unimi.dsi.fastutil.ints.IntSortedSet;
+
+public class IntSortedBitSet extends AbstractIntSortedSet implements Serializable {
+    @Serial
+    private static final long serialVersionUID = 1L;
+    private static final int BITS_PER_ELEMENT = Long.SIZE;
+    private long[] storage;
+
+    public IntSortedBitSet(int initialMaxValue) {
+        storage = new long[initialMaxValue / BITS_PER_ELEMENT + 1];
+    }
+
+    public IntSortedBitSet() {
+        this(1);
+    }
+
+    public IntSortedBitSet(IntCollection initial) {
+        this();
+        addAll(initial);
+    }
+
+    public IntSortedBitSet(int[] initial) {
+        this(Math.max(initial.length - 1, 1));
+        for (int value : initial) {
+            add(value);
+        }
+    }
+
+    @Override
+    public boolean contains(int k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        return storage.length > elementIndex && (storage[elementIndex] & 1L << bitIndex) != 0;
+    }
+
+    private void checkRange(int k) {
+        if (k < 0) {
+            throw new IllegalArgumentException("negative values not supported");
+        }
+    }
+
+    @Override
+    public boolean add(int k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        ensureCapacity(elementIndex);
+        boolean absent = (storage[elementIndex] & 1L << bitIndex) == 0;
+        storage[elementIndex] |= 1L << bitIndex;
+        return absent;
+    }
+
+    @Override
+    public boolean remove(int k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        if (storage.length <= elementIndex) {
+            return false;
+        }
+        boolean present = (storage[elementIndex] & 1L << bitIndex) != 0;
+        storage[elementIndex] &= ~(1L << bitIndex);
+        return present;
+    }
+
+    @Override
+    public void clear() {
+        Arrays.fill(storage, 0L);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o.getClass() == getClass()) {
+            return Arrays.equals(storage, ((IntSortedBitSet) o).storage);
+        }
+        return super.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(storage);
+    }
+
+    private void ensureCapacity(int index) {
+        if (storage.length <= index) {
+            long[] newStorage = new long[index + 1];
+            System.arraycopy(storage, 0, newStorage, 0, storage.length);
+            storage = newStorage;
+        }
+    }
+
+    @Override
+    public IntSortedSet subSet(int fromElement, int toElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public IntSortedSet headSet(int toElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public IntSortedSet tailSet(int fromElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public IntComparator comparator() {
+        return IntComparators.NATURAL_COMPARATOR;
+    }
+
+    @Override
+    public int firstInt() {
+        for (int index = 0; index < storage.length; index++) {
+            if (storage[index] != 0) {
+                return (int) (index * BITS_PER_ELEMENT
+                        + (int) MathUtil.log2Unsigned(Long.lowestOneBit(storage[index])));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int lastInt() {
+        for (int index = storage.length - 1; index >= 0; index--) {
+            if (storage[index] != 0) {
+                return index * BITS_PER_ELEMENT + (int) MathUtil.log2Unsigned(Long.highestOneBit(storage[index]));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int size() {
+        int size = 0;
+        for (long element : storage) {
+            size += Long.bitCount(element);
+        }
+        return size;
+    }
+
+    @Override
+    public IntBidirectionalIterator iterator() {
+        final int firstInt = firstInt();
+        return iterator(firstInt, firstInt);
+    }
+
+    @Override
+    public IntBidirectionalIterator iterator(int fromElement) {
+        return iterator(firstInt(), fromElement);
+    }
+
+    @Override
+    public String toString() {
+        return IntUtil.toCompactString(iterator());
+    }
+
+    private IntBidirectionalIterator iterator(final int first, final int fromElement) {
+        return new IntBidirectionalIterator() {
+            final int last = lastInt();
+            int position = fromElement;
+            int lastReturned = -1;
+
+            @Override
+            public int previousInt() {
+                for (; position >= first; position--) {
+                    if ((storage[position / BITS_PER_ELEMENT] & 1L << (position % BITS_PER_ELEMENT)) != 0) {
+                        lastReturned = position;
+                        return position--;
+                    }
+                }
+                throw new NoSuchElementException();
+            }
+
+            @Override
+            public boolean hasPrevious() {
+                return first >= 0 && position >= first;
+            }
+
+            @Override
+            public int nextInt() {
+                for (; position <= last; position++) {
+                    if ((storage[position / BITS_PER_ELEMENT] & 1L << (position % BITS_PER_ELEMENT)) != 0) {
+                        lastReturned = position;
+                        return position++;
+                    }
+                }
+                throw new NoSuchElementException();
+            }
+
+            @Override
+            public boolean hasNext() {
+                return last >= 0 && position <= last;
+            }
+
+            @Override
+            public void remove() {
+                IntSortedBitSet.this.remove(lastReturned);
+            }
+        };
+    }
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntUtil.java
new file mode 100644
index 0000000000..580f98c6e1
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/IntUtil.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.util.NoSuchElementException;
+
+import it.unimi.dsi.fastutil.ints.IntIterator;
+
+public class IntUtil {
+    private IntUtil() {
+        throw new AssertionError("do not instantiate");
+    }
+
+    /**
+     * Returns a compact string representation of the supplied {@link IntIterator}, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    public static String toCompactString(IntIterator iter) {
+        return toCompactString(iter, 0);
+    }
+
+    /**
+     * Returns a compact string representation of the supplied {@link IntIterator}, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    static String toCompactString(IntIterator iter, int delta) {
+        if (!iter.hasNext()) {
+            return "[]";
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append('[');
+        appendCompact(iter, builder, delta);
+        builder.append(']');
+        return builder.toString();
+    }
+
+    /**
+     * Returns a compact string representation of the supplied int array, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    public static String toCompactString(int[] iter) {
+        return toCompactString(wrap(iter));
+    }
+
+    /**
+     * Appends the contents of the supplied {@link IntIterator} to the {@link StringBuilder} instance,
+     * comma-delimited, collapsing ranges together with a hyphen (-).  Only provides reasonable
+     * results if the contents of the iterator are sorted.
+     */
+    public static void appendCompact(IntIterator iter, StringBuilder builder) {
+        appendCompact(iter, builder, 0);
+    }
+
+    /**
+     * Appends the contents of the supplied {@link IntIterator} to the {@link StringBuilder} instance,
+     * comma-delimited, collapsing ranges together with a hyphen (-).  Only provides reasonable
+     * results if the contents of the iterator are sorted.
+     */
+    static void appendCompact(IntIterator iter, StringBuilder builder, int outputDelta) {
+        int rangeStart = iter.nextInt();
+        builder.append(rangeStart + outputDelta);
+        int current = rangeStart;
+        int prev = current;
+        while (iter.hasNext()) {
+            current = iter.nextInt();
+            if (current != prev + 1) {
+                // end any range we were in:
+                if (rangeStart != prev) {
+                    builder.append('-').append(prev + outputDelta);
+                }
+                builder.append(",").append(current + outputDelta);
+                rangeStart = current;
+            }
+            prev = current;
+        }
+        if (rangeStart != prev) {
+            builder.append('-').append(prev + outputDelta);
+        }
+    }
+
+    public static IntIterator wrap(int... ints) {
+        return new IntIterator() {
+            int index = 0;
+
+            @Override
+            public int nextInt() {
+                try {
+                    return ints[index++];
+                } catch (ArrayIndexOutOfBoundsException e) {
+                    throw new NoSuchElementException();
+                }
+            }
+
+            @Override
+            public boolean hasNext() {
+                return index < ints.length;
+            }
+        };
+    }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/MathUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/MathUtil.java
new file mode 100644
index 0000000000..c435427f7b
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/MathUtil.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import it.unimi.dsi.fastutil.longs.Long2ByteMap;
+import it.unimi.dsi.fastutil.longs.Long2ByteOpenHashMap;
+
+public class MathUtil {
+
+    private static final Long2ByteMap LOG2_MAP;
+
+    static {
+        LOG2_MAP = new Long2ByteOpenHashMap();
+        for (byte i = 0; i < Long.SIZE; i++) {
+            LOG2_MAP.put(1L << i, i);
+        }
+    }
+
+    private MathUtil() {
+    }
+
+    public static long maxUnsigned(long a, long b) {
+        return Long.compareUnsigned(a, b) > 0 ? a : b;
+    }
+
+    public static long minUnsigned(long a, long b) {
+        return Long.compareUnsigned(a, b) < 0 ? a : b;
+    }
+
+    public static long log2Unsigned(long value) {
+        final byte result = LOG2_MAP.getOrDefault(value, Byte.MIN_VALUE);
+        if (result < 0) {
+            throw new IllegalArgumentException("cannot resolve log2 value for " + Long.toUnsignedString(value, 16));
+        }
+        return result;
+    }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
new file mode 100644
index 0000000000..45aaa49150
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.io.Serial;
+import java.io.Serializable;
+
+import it.unimi.dsi.fastutil.ints.IntIterator;
+import it.unimi.dsi.fastutil.ints.IntSortedSet;
+
+/**
+ * A specialized bit set for storing partition IDs.
+ * Partition IDs are in the range [-1, 32766] (inclusive).
+ * The value -1 is used to represent the "unpartitioned" partition.
+ * The value 32767 is reserved for the internal shift and cannot be used as a partition ID.
+ * This class internally shifts the partition IDs by +1 to fit into an unsigned short range [0, 32767].
+ */
+public class Partitions implements Serializable {
+    @Serial
+    private static final long serialVersionUID = 1L;
+    private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1;
+    private static final int MIN_PARTITION_ID = -1;
+    private static final short MINUS_ONE = (short) -1;
+    private final IntSortedSet delegate;
+
+    public Partitions() {
+        delegate = new IntSortedBitSet();
+    }
+
+    public Partitions(int initialMaxValue) {
+        delegate = new IntSortedBitSet(initialMaxValue + 1);
+    }
+
+    public Partitions(IntSortedSet delegate) {
+        this.delegate = delegate;
+    }
+
+    /**
+     * Adds a partition ID to the set. Partition ID must be in the range [-1, 32766].
+     * @param k the partition ID to add
+     * @return true if the partition ID was added, false if it was already present
+     * @throws IllegalArgumentException if the partition ID is out of range
+     */
+    public boolean add(int k) {
+        if (k > MAX_PARTITION_ID || k < MIN_PARTITION_ID) {
+            throw new IllegalArgumentException("Partition number " + k + " out of range");
+        }
+        return delegate.add((short) (k + 1));
+    }
+
+    public boolean isEmpty() {
+        return delegate.isEmpty();
+    }
+
+    public void clear() {
+        delegate.clear();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (!(o instanceof Partitions other)) {
+            return false;
+        }
+        return delegate.equals(other.delegate);
+    }
+
+    @Override
+    public int hashCode() {
+        return delegate.hashCode();
+    }
+
+    @Override
+    public String toString() {
+        if (isEmpty()) {
+            return "[]";
+        }
+        IntIterator iter = delegate.iterator();
+        StringBuilder builder = new StringBuilder();
+        builder.append('[');
+        if (delegate.firstInt() == 0) {
+            builder.append(MINUS_ONE);
+            iter.nextInt(); // this is the zero we just printed
+            if (iter.hasNext()) {
+                builder.append(',');
+            }
+        }
+        IntUtil.appendCompact(iter, builder, MINUS_ONE);
+        builder.append(']');
+        return builder.toString();
+    }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortSortedBitSet.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortSortedBitSet.java
new file mode 100644
index 0000000000..4096f794c9
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortSortedBitSet.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.NoSuchElementException;
+
+import it.unimi.dsi.fastutil.shorts.AbstractShortSortedSet;
+import it.unimi.dsi.fastutil.shorts.ShortBidirectionalIterator;
+import it.unimi.dsi.fastutil.shorts.ShortCollection;
+import it.unimi.dsi.fastutil.shorts.ShortComparator;
+import it.unimi.dsi.fastutil.shorts.ShortComparators;
+import it.unimi.dsi.fastutil.shorts.ShortSortedSet;
+
+public class ShortSortedBitSet extends AbstractShortSortedSet implements Serializable {
+    @Serial
+    private static final long serialVersionUID = 1L;
+    private static final short BITS_PER_ELEMENT = Long.SIZE;
+    private long[] storage;
+
+    public ShortSortedBitSet(int initialMaxValue) {
+        storage = new long[initialMaxValue / BITS_PER_ELEMENT + 1];
+    }
+
+    public ShortSortedBitSet() {
+        this(1);
+    }
+
+    public ShortSortedBitSet(ShortCollection initial) {
+        this();
+        addAll(initial);
+    }
+
+    public ShortSortedBitSet(short[] initial) {
+        this(Math.max(initial.length - 1, 1));
+        for (short value : initial) {
+            add(value);
+        }
+    }
+
+    @Override
+    public boolean contains(short k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        return storage.length > elementIndex && (storage[elementIndex] & 1L << bitIndex) != 0;
+    }
+
+    private void checkRange(int k) {
+        if (k < 0) {
+            throw new IllegalArgumentException("negative values not supported");
+        }
+    }
+
+    @Override
+    public boolean add(short k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        ensureCapacity(elementIndex);
+        boolean absent = (storage[elementIndex] & 1L << bitIndex) == 0;
+        storage[elementIndex] |= 1L << bitIndex;
+        return absent;
+    }
+
+    @Override
+    public boolean remove(short k) {
+        checkRange(k);
+        int elementIndex = k / BITS_PER_ELEMENT;
+        int bitIndex = k % BITS_PER_ELEMENT;
+        if (storage.length <= elementIndex) {
+            return false;
+        }
+        boolean present = (storage[elementIndex] & 1L << bitIndex) != 0;
+        storage[elementIndex] &= ~(1L << bitIndex);
+        return present;
+    }
+
+    @Override
+    public void clear() {
+        Arrays.fill(storage, 0L);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o.getClass() == getClass()) {
+            return Arrays.equals(storage, ((ShortSortedBitSet) o).storage);
+        }
+        return super.equals(o);
+    }
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode(storage);
+    }
+
+    private void ensureCapacity(int index) {
+        if (storage.length <= index) {
+            long[] newStorage = new long[index + 1];
+            System.arraycopy(storage, 0, newStorage, 0, storage.length);
+            storage = newStorage;
+        }
+    }
+
+    @Override
+    public ShortSortedSet subSet(short fromElement, short toElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public ShortSortedSet headSet(short toElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public ShortSortedSet tailSet(short fromElement) {
+        throw new UnsupportedOperationException("nyi");
+    }
+
+    @Override
+    public ShortComparator comparator() {
+        return ShortComparators.NATURAL_COMPARATOR;
+    }
+
+    @Override
+    public short firstShort() {
+        for (int index = 0; index < storage.length; index++) {
+            if (storage[index] != 0) {
+                return (short) (index * BITS_PER_ELEMENT
+                        + (short) MathUtil.log2Unsigned(Long.lowestOneBit(storage[index])));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public short lastShort() {
+        for (int index = storage.length - 1; index >= 0; index--) {
+            if (storage[index] != 0) {
+                return (short) (index * BITS_PER_ELEMENT
+                        + (short) MathUtil.log2Unsigned(Long.highestOneBit(storage[index])));
+            }
+        }
+        return -1;
+    }
+
+    @Override
+    public int size() {
+        int size = 0;
+        for (long element : storage) {
+            size += Long.bitCount(element);
+        }
+        return size;
+    }
+
+    @Override
+    public ShortBidirectionalIterator iterator() {
+        final short firstShort = firstShort();
+        return iterator(firstShort, firstShort);
+    }
+
+    @Override
+    public ShortBidirectionalIterator iterator(short fromElement) {
+        return iterator(firstShort(), fromElement);
+    }
+
+    @Override
+    public String toString() {
+        return ShortUtil.toCompactString(iterator());
+    }
+
+    private ShortBidirectionalIterator iterator(final short first, final short fromElement) {
+        return new ShortBidirectionalIterator() {
+            final short last = lastShort();
+            short position = fromElement;
+            short lastReturned = -1;
+
+            @Override
+            public short previousShort() {
+                for (; position >= first; position--) {
+                    if ((storage[position / BITS_PER_ELEMENT] & 1L << (position % BITS_PER_ELEMENT)) != 0) {
+                        lastReturned = position;
+                        return position--;
+                    }
+                }
+                throw new NoSuchElementException();
+            }
+
+            @Override
+            public boolean hasPrevious() {
+                return first >= 0 && position >= first;
+            }
+
+            @Override
+            public short nextShort() {
+                for (; position <= last; position++) {
+                    if ((storage[position / BITS_PER_ELEMENT] & 1L << (position % BITS_PER_ELEMENT)) != 0) {
+                        lastReturned = position;
+                        return position++;
+                    }
+                }
+                throw new NoSuchElementException();
+            }
+
+            @Override
+            public boolean hasNext() {
+                return last >= 0 && position <= last;
+            }
+
+            @Override
+            public void remove() {
+                ShortSortedBitSet.this.remove(lastReturned);
+            }
+        };
+    }
+
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortUtil.java
new file mode 100644
index 0000000000..e5df8d3406
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/ShortUtil.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.utils;
+
+import java.util.NoSuchElementException;
+
+import it.unimi.dsi.fastutil.shorts.ShortIterator;
+
+public class ShortUtil {
+    private ShortUtil() {
+        throw new AssertionError("do not instantiate");
+    }
+
+    /**
+     * Returns a compact string representation of the supplied {@link ShortIterator}, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    public static String toCompactString(ShortIterator iter) {
+        return toCompactString(iter, (short) 0);
+    }
+
+    /**
+     * Returns a compact string representation of the supplied {@link ShortIterator}, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    static String toCompactString(ShortIterator iter, short delta) {
+        if (!iter.hasNext()) {
+            return "[]";
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append('[');
+        appendCompact(iter, builder, delta);
+        builder.append(']');
+        return builder.toString();
+    }
+
+    /**
+     * Returns a compact string representation of the supplied short array, enclosed
+     * by square-brackets ([]), comma-delimited, collapsing ranges together with a hyphen (-).
+     * Only provides reasonable results if the contents of the iterator are sorted.
+     */
+    public static String toCompactString(short[] iter) {
+        return toCompactString(wrap(iter));
+    }
+
+    /**
+     * Appends the contents of the supplied {@link ShortIterator} to the {@link StringBuilder} instance,
+     * comma-delimited, collapsing ranges together with a hyphen (-).  Only provides reasonable
+     * results if the contents of the iterator are sorted.
+     */
+    public static void appendCompact(ShortIterator iter, StringBuilder builder) {
+        appendCompact(iter, builder, (short) 0);
+    }
+
+    /**
+     * Appends the contents of the supplied {@link ShortIterator} to the {@link StringBuilder} instance,
+     * comma-delimited, collapsing ranges together with a hyphen (-).  Only provides reasonable
+     * results if the contents of the iterator are sorted.
+     */
+    static void appendCompact(ShortIterator iter, StringBuilder builder, short outputDelta) {
+        if (!iter.hasNext()) {
+            return;
+        }
+        short rangeStart = iter.nextShort();
+        builder.append(rangeStart + outputDelta);
+        short current = rangeStart;
+        short prev = current;
+        while (iter.hasNext()) {
+            current = iter.nextShort();
+            if (current != prev + 1) {
+                // end any range we were in:
+                if (rangeStart != prev) {
+                    builder.append('-').append(prev + outputDelta);
+                }
+                builder.append(",").append(current + outputDelta);
+                rangeStart = current;
+            }
+            prev = current;
+        }
+        if (rangeStart != prev) {
+            builder.append('-').append(prev + outputDelta);
+        }
+    }
+
+    public static ShortIterator wrap(short... shorts) {
+        return new ShortIterator() {
+            int index = 0;
+
+            @Override
+            public short nextShort() {
+                try {
+                    return shorts[index++];
+                } catch (ArrayIndexOutOfBoundsException e) {
+                    throw new NoSuchElementException();
+                }
+            }
+
+            @Override
+            public boolean hasNext() {
+                return index < shorts.length;
+            }
+        };
+    }
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/MathUtilTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/MathUtilTest.java
new file mode 100644
index 0000000000..70c77b763e
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/MathUtilTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.utils;
+
+import org.apache.asterix.common.utils.MathUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class MathUtilTest {
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final int BENCHMARK_ITERATIONS = 5000000;
+    private final double LOG_2 = Math.log(2);
+
+    @Test
+    public void test() {
+        for (int i = 0; i < Long.SIZE; i++) {
+            long value = 1L << i;
+            Assert.assertEquals(Long.toUnsignedString(value, 16),
+                    value == Long.MIN_VALUE ? Long.SIZE - 1 : (long) (Math.log(value) / LOG_2),
+                    MathUtil.log2Unsigned(value));
+        }
+    }
+
+    @Test
+    public void benchmarkLookupTable() {
+        lookupTableRun();
+        lookupTableRun();
+    }
+
+    @Test
+    public void benchmarkMathLog() {
+        mathLogRun();
+        mathLogRun();
+    }
+
+    private void lookupTableRun() {
+        long start = System.nanoTime();
+        for (int j = 0; j < BENCHMARK_ITERATIONS; j++) {
+            for (int i = 0; i < Long.SIZE; i++) {
+                final long value = 1L << i;
+                MathUtil.log2Unsigned(value);
+            }
+        }
+        LOGGER.info("total time for {} iterations of lookup table: {}ns", BENCHMARK_ITERATIONS * Long.SIZE,
+                System.nanoTime() - start);
+    }
+
+    private void mathLogRun() {
+        long ignored = 0;
+        long start = System.nanoTime();
+        for (int j = 0; j < BENCHMARK_ITERATIONS; j++) {
+            for (int i = 0; i < Long.SIZE; i++) {
+                final long value = 1L << i;
+                ignored += (long) (Math.log(value) / LOG_2);
+            }
+        }
+        LOGGER.info("total time for {} iterations of Math.log(n) / Math.log(2): {}ns", BENCHMARK_ITERATIONS * Long.SIZE,
+                System.nanoTime() - start);
+    }
+}
diff --git a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/ShortSortedBitSetTest.java b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/ShortSortedBitSetTest.java
new file mode 100644
index 0000000000..561e647785
--- /dev/null
+++ b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/utils/ShortSortedBitSetTest.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.utils;
+
+import java.util.Random;
+
+import org.apache.asterix.common.utils.ShortSortedBitSet;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+import it.unimi.dsi.fastutil.shorts.ShortIterator;
+import it.unimi.dsi.fastutil.shorts.ShortSortedSet;
+
+public class ShortSortedBitSetTest {
+    private static final Logger LOGGER = LogManager.getLogger();
+
+    @Test
+    public void test() {
+        Random r = new Random();
+        ShortSortedSet set = new ShortSortedBitSet();
+        short uniques = 0;
+        for (short i = 0; i < 500; i++) {
+            uniques += set.add((short) r.nextInt(1000)) ? 1 : 0;
+        }
+        LOGGER.info("set {}", set);
+        LOGGER.info("set contains {} unique elements", uniques);
+        Assert.assertEquals("set size() != number of uniques", uniques, set.size());
+        for (short values : set) {
+            uniques--;
+        }
+        Assert.assertEquals("iteration count != number of uniques", 0, uniques);
+        int size = set.size();
+        short removed = 0;
+        for (ShortIterator iter = set.iterator(); iter.hasNext();) {
+            iter.nextShort();
+            iter.remove();
+            removed++;
+        }
+        Assert.assertEquals("removed count != before size", size, removed);
+        Assert.assertEquals("new size != 0", 0, set.size());
+        for (ShortIterator iter = set.iterator(); iter.hasNext();) {
+            Assert.fail("got iteration on empty set: " + iter.nextShort());
+        }
+        for (short value : set) {
+            Assert.fail("got iteration on empty set: " + value);
+        }
+    }
+
+    @Test
+    public void testSingletonSets() {
+        for (short i = 0; i <= 16384; i++) {
+            testSingletonSet(i);
+        }
+    }
+
+    private void testSingletonSet(short key) {
+        LOGGER.log(key % 1024 == 0 ? Level.INFO : Level.DEBUG, "testing {}", key);
+        ShortSortedSet set = new ShortSortedBitSet();
+        set.add(key);
+        final int size = set.size();
+        LOGGER.debug("singleton set {} contains {} unique elements", set, size);
+        Assert.assertEquals("singleton size != 1", 1, size);
+        for (short values : set) {
+            LOGGER.debug("got value: {}", values);
+        }
+        int removed = 0;
+        for (ShortIterator iter = set.iterator(); iter.hasNext();) {
+            iter.nextShort();
+            iter.remove();
+            removed++;
+        }
+        Assert.assertEquals("removed count != before size", size, removed);
+        Assert.assertEquals("new size != 0", 0, set.size());
+    }
+
+    @Test
+    public void emptySetTest() {
+        ShortSortedSet set = new ShortSortedBitSet();
+        LOGGER.info("set {}", set);
+        LOGGER.info("set contains {} unique elements", set.size());
+        for (ShortIterator iter = set.iterator(); iter.hasNext();) {
+            Assert.fail("got iteration on empty set: " + iter.nextShort());
+        }
+        for (short value : set) {
+            Assert.fail("got iteration on empty set: " + value);
+        }
+    }
+}


(asterixdb) 13/25: [ASTERIXDB-3392][EXT] Default to STRING for null-typed fields during schema inference in COPY TO parquet

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 384570e47579980735a593682a451266c3f4830a
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Fri Sep 19 11:52:59 2025 +0530

    [ASTERIXDB-3392][EXT] Default to STRING for null-typed fields during schema inference in COPY TO parquet
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    When a compute partition consists entirely of null values, schema inference in copy to parquet defaults to STRING type in parquet.
    
    Ext-ref: MB-68610
    Change-Id: I60069bf3d44c60effb69e5555bbf7e869327cc5b
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20395
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Ali Alsuliman <al...@gmail.com>
---
 .../parquet-error-checks.23.ddl.sqlpp              |  1 +
 .../parquet-null1.01.ddl.sqlpp}                    |  3 +++
 .../parquet-null1.02.update.sqlpp}                 | 25 +++++++++++-----------
 .../parquet-null1.03.ddl.sqlpp}                    | 15 +++++++------
 .../parquet-null1.04.query.sqlpp}                  |  8 +++----
 .../parquet-null2.01.ddl.sqlpp}                    |  5 ++++-
 .../parquet-null2.02.update.sqlpp}                 | 17 ++++++++++-----
 .../parquet-null2.03.ddl.sqlpp}                    | 15 +++++++------
 .../parquet-null2.04.query.sqlpp}                  |  8 +++----
 .../parquet-simple/parquet-simple.01.ddl.sqlpp     |  2 ++
 .../parquet-simple/parquet-simple.02.update.sqlpp  |  4 ++--
 .../copy-to/parquet-null1/parquet-null1.04.adm     |  2 ++
 .../copy-to/parquet-null2/parquet-null2.04.adm     |  3 +++
 .../runtimets/testsuite_external_dataset_s3.xml    | 21 +++++++++++++++++-
 .../writer/printer/parquet/ParquetSchemaTree.java  | 21 ++++++++++++------
 .../printer/parquet/SchemaCheckerLazyVisitor.java  |  6 +++---
 16 files changed, 104 insertions(+), 52 deletions(-)

diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
index a2dbe7f51d..f504fd5d57 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
@@ -23,6 +23,7 @@ USE test;
 
 CREATE COLLECTION TestCollection6(ColumnType1) PRIMARY KEY id;
 CREATE COLLECTION TestCollection7(ColumnType1) PRIMARY KEY id;
+CREATE COLLECTION TestCollection8(ColumnType1) PRIMARY KEY id;
 
 
 
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.01.ddl.sqlpp
similarity index 95%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.01.ddl.sqlpp
index 76970a5579..0f80378feb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.01.ddl.sqlpp
@@ -23,3 +23,6 @@ USE test;
 
 CREATE TYPE ColumnType2 AS {
 };
+
+
+CREATE DATASET col2 primary key(id:int);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.02.update.sqlpp
similarity index 80%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.02.update.sqlpp
index be198a29d3..3d5158ba8e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.15.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.02.update.sqlpp
@@ -20,22 +20,21 @@
 USE test;
 
 
-
-
-
-
-insert into TestCollection({"id":1, "name": []});
-
-
+INSERT INTO col2 (
+    [
+{"id": 1, "name": "aqay awil"},
+    {"id": 2}
+    ]
+);
 
 COPY (
-select c.* from TestCollection c
-    ) toWriter
+   select * from col2
+) toWriter
 TO %adapter%
-PATH (%pathprefix% "copy-to-result", "parquet-error-checks15")
+PATH (%pathprefix% "copy-to-result", "parquet-null1")
 WITH {
     %template_colons%,
     %additionalProperties%
-    "format":"parquet"
-    }
-
+    "format":"parquet",
+    "version" : "2"
+};
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.03.ddl.sqlpp
similarity index 75%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.03.ddl.sqlpp
index a2dbe7f51d..ab296ea796 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.03.ddl.sqlpp
@@ -20,9 +20,12 @@
 USE test;
 
 
-
-CREATE COLLECTION TestCollection6(ColumnType1) PRIMARY KEY id;
-CREATE COLLECTION TestCollection7(ColumnType1) PRIMARY KEY id;
-
-
-
+CREATE EXTERNAL DATASET DatasetCopy(ColumnType2) USING %adapter%
+(
+  %template%,
+  %additional_Properties%,
+  ("definition"="%path_prefix%copy-to-result/parquet-null1"),
+  ("format" = "parquet"),
+  ("requireVersionChangeDetection"="false"),
+  ("include"="*.parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.04.query.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.04.query.sqlpp
index 76970a5579..22540697a1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null1/parquet-null1.04.query.sqlpp
@@ -17,9 +17,9 @@
  * under the License.
  */
 
-DROP DATAVERSE test if exists;
-CREATE DATAVERSE test;
 USE test;
 
-CREATE TYPE ColumnType2 AS {
-};
+
+SELECT *
+FROM DatasetCopy c
+ORDER BY c.col2.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.01.ddl.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.01.ddl.sqlpp
index 76970a5579..8bac75ecc2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.01.ddl.sqlpp
@@ -22,4 +22,7 @@ CREATE DATAVERSE test;
 USE test;
 
 CREATE TYPE ColumnType2 AS {
-};
+    };
+
+
+CREATE DATASET col2 primary key(id:int);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.02.update.sqlpp
similarity index 78%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.02.update.sqlpp
index 9b21be7bfb..ceafc41736 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.02.update.sqlpp
@@ -20,15 +20,22 @@
 USE test;
 
 
+INSERT INTO col2 (
+    [
+{"id": 1, "name": "aqay awil", "centuries" : []},
+    {"id": 2, "centuries" : []},
+    {"id":3 , "name": null, "centuries" : []}
+    ]
+);
+
 COPY (
-   select "123" as id
-) toWriter
+select * from col2
+    ) toWriter
 TO %adapter%
-PATH (%pathprefix% "copy-to-result", "parquet-simple")
-TYPE ( {id:string} )
+PATH (%pathprefix% "copy-to-result", "parquet-null2")
 WITH {
     %template_colons%,
     %additionalProperties%
     "format":"parquet",
     "version" : "2"
-};
\ No newline at end of file
+    };
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.03.ddl.sqlpp
similarity index 75%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.03.ddl.sqlpp
index a2dbe7f51d..2e6e1809ed 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/parquet-error-checks/parquet-error-checks.23.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.03.ddl.sqlpp
@@ -20,9 +20,12 @@
 USE test;
 
 
-
-CREATE COLLECTION TestCollection6(ColumnType1) PRIMARY KEY id;
-CREATE COLLECTION TestCollection7(ColumnType1) PRIMARY KEY id;
-
-
-
+CREATE EXTERNAL DATASET DatasetCopy(ColumnType2) USING %adapter%
+(
+  %template%,
+  %additional_Properties%,
+  ("definition"="%path_prefix%copy-to-result/parquet-null2"),
+  ("format" = "parquet"),
+  ("requireVersionChangeDetection"="false"),
+  ("include"="*.parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.04.query.sqlpp
similarity index 90%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.04.query.sqlpp
index 76970a5579..22540697a1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-null2/parquet-null2.04.query.sqlpp
@@ -17,9 +17,9 @@
  * under the License.
  */
 
-DROP DATAVERSE test if exists;
-CREATE DATAVERSE test;
 USE test;
 
-CREATE TYPE ColumnType2 AS {
-};
+
+SELECT *
+FROM DatasetCopy c
+ORDER BY c.col2.id;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
index 76970a5579..abff4f0231 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.01.ddl.sqlpp
@@ -23,3 +23,5 @@ USE test;
 
 CREATE TYPE ColumnType2 AS {
 };
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp
index 9b21be7bfb..d598e52862 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/parquet-simple/parquet-simple.02.update.sqlpp
@@ -21,8 +21,8 @@ USE test;
 
 
 COPY (
-   select "123" as id
-) toWriter
+select "123" as id
+    ) toWriter
 TO %adapter%
 PATH (%pathprefix% "copy-to-result", "parquet-simple")
 TYPE ( {id:string} )
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null1/parquet-null1.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null1/parquet-null1.04.adm
new file mode 100644
index 0000000000..c1aed87d1a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null1/parquet-null1.04.adm
@@ -0,0 +1,2 @@
+{ "c": { "col2": { "name": "aqay awil", "id": 1 } } }
+{ "c": { "col2": { "id": 2 } } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null2/parquet-null2.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null2/parquet-null2.04.adm
new file mode 100644
index 0000000000..29ca9ecd0d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/parquet-null2/parquet-null2.04.adm
@@ -0,0 +1,3 @@
+{ "c": { "col2": { "centuries": [  ], "name": "aqay awil", "id": 1 } } }
+{ "c": { "col2": { "centuries": [  ], "id": 2 } } }
+{ "c": { "col2": { "centuries": [  ], "id": 3 } } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 5b1d265e20..a2677833b9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -69,6 +69,26 @@
         <output-dir compare="Text">parquet-simple</output-dir>
       </compilation-unit>
     </test-case>
+    <test-case FilePath="copy-to">
+      <compilation-unit name="parquet-null1">
+        <placeholder name="adapter" value="S3" />
+        <placeholder name="pathprefix" value="" />
+        <placeholder name="path_prefix" value="" />
+        <placeholder name="additionalProperties" value='"container":"playground",' />
+        <placeholder name="additional_Properties" value='("container"="playground")' />
+        <output-dir compare="Text">parquet-null1</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="copy-to">
+      <compilation-unit name="parquet-null2">
+        <placeholder name="adapter" value="S3" />
+        <placeholder name="pathprefix" value="" />
+        <placeholder name="path_prefix" value="" />
+        <placeholder name="additionalProperties" value='"container":"playground",' />
+        <placeholder name="additional_Properties" value='("container"="playground")' />
+        <output-dir compare="Text">parquet-null2</output-dir>
+      </compilation-unit>
+    </test-case>
     <test-case FilePath="copy-to">
       <compilation-unit name="parquet-tweet">
         <placeholder name="adapter" value="S3" />
@@ -278,7 +298,6 @@
         <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'. Supported values: [1, 2]</expected-error>
         <expected-error>ASX0039: Expected integer value, got yvghc (in line 22, at column 6)</expected-error>
         <expected-error>ASX1209: Maximum value allowed for 'max-schemas' is 10. Found 15</expected-error>
-        <expected-error>HYR0133: Schema could not be inferred, empty types found in the result</expected-error>
         <expected-error>HYR0134: Schema Limit exceeded, maximum number of heterogeneous schemas allowed : '2'</expected-error>
         <expected-error>ASX1204: 'rectangle' type not supported in parquet format</expected-error>
         <expected-error>ASX0072: Parquet does not support arrays containing mixed data types</expected-error>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/ParquetSchemaTree.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/ParquetSchemaTree.java
index 5fb2ac7c6e..a76426232f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/ParquetSchemaTree.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/ParquetSchemaTree.java
@@ -38,6 +38,12 @@ import org.apache.parquet.schema.Types;
 public class ParquetSchemaTree {
     private static final Logger LOGGER = LogManager.getLogger();
 
+    private static final SchemaNode DEFAULT_SCHEMA_NODE_FOR_NULL = new SchemaNode();
+    private static final FlatType DEFAULT_TYPE_TAG_FOR_NULL = new FlatType(ATypeTag.STRING);
+    static {
+        DEFAULT_SCHEMA_NODE_FOR_NULL.setType(DEFAULT_TYPE_TAG_FOR_NULL);
+    }
+
     public static class SchemaNode {
         private AbstractType type;
 
@@ -179,12 +185,13 @@ public class ParquetSchemaTree {
     public static void buildParquetSchema(Types.Builder builder, SchemaNode schemaNode, String columnName)
             throws HyracksDataException {
         if (schemaNode.getType() == null) {
-            LOGGER.info(
-                    "Child type not set for record value with column name: " + LogRedactionUtil.userData(columnName));
-            throw new HyracksDataException(ErrorCode.EMPTY_TYPE_INFERRED);
+            buildFlat(builder, DEFAULT_TYPE_TAG_FOR_NULL, columnName);
+            return;
         }
         AbstractType typeClass = schemaNode.getType();
-        if (typeClass instanceof RecordType) {
+        if (typeClass == null) {
+            buildFlat(builder, DEFAULT_TYPE_TAG_FOR_NULL, columnName);
+        } else if (typeClass instanceof RecordType) {
             buildRecord(builder, (RecordType) schemaNode.getType(), columnName);
         } else if (typeClass instanceof ListType) {
             buildList(builder, (ListType) schemaNode.getType(), columnName);
@@ -206,10 +213,10 @@ public class ParquetSchemaTree {
         Types.BaseListBuilder<?, ?> childBuilder = getListChild(builder);
         SchemaNode child = type.child;
         if (child == null) {
-            LOGGER.info("Child type not set for list with column name: " + LogRedactionUtil.userData(columnName));
-            throw new HyracksDataException(ErrorCode.EMPTY_TYPE_INFERRED);
+            buildParquetSchema(childBuilder, DEFAULT_SCHEMA_NODE_FOR_NULL, columnName);
+        } else {
+            buildParquetSchema(childBuilder, child, columnName);
         }
-        buildParquetSchema(childBuilder, child, columnName);
     }
 
     private static void buildFlat(Types.Builder builder, FlatType type, String columnName) throws HyracksDataException {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/SchemaCheckerLazyVisitor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/SchemaCheckerLazyVisitor.java
index 6e03144ac9..5e911a6ba2 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/SchemaCheckerLazyVisitor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/printer/parquet/SchemaCheckerLazyVisitor.java
@@ -113,13 +113,13 @@ public class SchemaCheckerLazyVisitor implements ISchemaChecker,
     @Override
     public ISchemaChecker.SchemaComparisonType visit(FlatLazyVisitablePointable currentValue,
             ParquetSchemaTree.SchemaNode schemaNode) throws HyracksDataException {
-        if (schemaNode.getType() == null) {
-            return SchemaComparisonType.GROWING;
-        }
         // SchemaNode.getTypeTag can never be MISSING here
         if (currentValue.getTypeTag() == ATypeTag.NULL) {
             return SchemaComparisonType.EQUIVALENT;
         }
+        if (schemaNode.getType() == null) {
+            return SchemaComparisonType.GROWING;
+        }
         if (!(schemaNode.getType() instanceof ParquetSchemaTree.FlatType inferredType)) {
             return ISchemaChecker.SchemaComparisonType.CONFLICTING;
         }


(asterixdb) 12/25: [NO ISSUE][*DB][STO] Increased log brevity in cloud storage

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 44adbba68d676ee2ed0d523a73138f0759af4c9e
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Oct 12 09:37:04 2025 -0400

    [NO ISSUE][*DB][STO] Increased log brevity in cloud storage
    
    Ext-ref: MB-68946
    Change-Id: I37332782d583d5c9daaf8b58dad7c4a1a2992bd6
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20483
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
 .../asterix/cloud/AbstractCloudIOManager.java      | 10 +++---
 .../cloud/lazy/accessor/InitialCloudAccessor.java  |  5 ++-
 .../lazy/accessor/ReplaceableCloudAccessor.java    |  5 +--
 .../lazy/accessor/SelectiveCloudAccessor.java      |  4 +--
 .../apache/asterix/cloud/util/CloudFileUtil.java   |  1 -
 .../apache/asterix/common/utils/Partitions.java    | 38 ++++++++++++++++++++--
 6 files changed, 47 insertions(+), 16 deletions(-)

diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 526af2df1c..7712f08653 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -31,7 +31,6 @@ import java.nio.file.FileStore;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -50,6 +49,7 @@ import org.apache.asterix.common.cloud.IPartitionBootstrapper;
 import org.apache.asterix.common.config.CloudProperties;
 import org.apache.asterix.common.metadata.MetadataConstants;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
@@ -82,12 +82,11 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
     protected final ICloudGuardian guardian;
     protected final IWriteBufferProvider writeBufferProvider;
     protected final String bucket;
-    protected final Set<Integer> partitions;
+    protected final Partitions partitions;
     protected final List<FileReference> partitionPaths;
     protected final IOManager localIoManager;
     protected final INamespacePathResolver nsPathResolver;
     private final List<FileStore> drivePaths;
-    private final String storageScheme;
 
     public AbstractCloudIOManager(IOManager ioManager, CloudProperties cloudProperties,
             INamespacePathResolver nsPathResolver, ICloudGuardian guardian) throws HyracksDataException {
@@ -99,11 +98,10 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
         this.guardian = guardian;
         int numOfThreads = getIODevices().size() * getIOParallelism();
         writeBufferProvider = new WriteBufferProvider(numOfThreads, cloudClient.getWriteBufferSize());
-        partitions = new HashSet<>();
+        partitions = new Partitions();
         partitionPaths = new ArrayList<>();
         this.localIoManager = ioManager;
         drivePaths = PhysicalDrive.getDrivePaths(ioDevices);
-        storageScheme = cloudProperties.getStorageScheme();
     }
 
     /*
@@ -179,7 +177,7 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
                 }
             }
         } else {
-            LOGGER.info("Cleaning node partitions...");
+            LOGGER.info("Cleaning node partitions {}...", partitions);
             for (FileReference partitionPath : partitionPaths) {
                 CloudFileUtil.cleanDirectoryFiles(localIoManager, cloudFiles, partitionPath);
             }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
index 8c3321ad86..5070d40bce 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/InitialCloudAccessor.java
@@ -18,10 +18,9 @@
  */
 package org.apache.asterix.cloud.lazy.accessor;
 
-import java.util.Collections;
-
 import org.apache.asterix.cloud.clients.ICloudClient;
 import org.apache.asterix.cloud.lazy.NoOpParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.control.nc.io.IOManager;
@@ -35,7 +34,7 @@ public class InitialCloudAccessor extends ReplaceableCloudAccessor {
     };
 
     public InitialCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager) {
-        super(cloudClient, bucket, localIoManager, Collections.emptySet(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
+        super(cloudClient, bucket, localIoManager, Partitions.empty(), NO_OP_REPLACER, NoOpParallelCacher.INSTANCE);
     }
 
     @Override
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
index 91046a1a06..8dbf605608 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/ReplaceableCloudAccessor.java
@@ -28,6 +28,7 @@ import org.apache.asterix.cloud.bulk.IBulkOperationCallBack;
 import org.apache.asterix.cloud.clients.CloudFile;
 import org.apache.asterix.cloud.clients.ICloudClient;
 import org.apache.asterix.cloud.lazy.IParallelCacher;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StorageConstants;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -42,13 +43,13 @@ import org.apache.logging.log4j.Logger;
  */
 public class ReplaceableCloudAccessor extends AbstractLazyAccessor {
     private static final Logger LOGGER = LogManager.getLogger();
-    private final Set<Integer> partitions;
+    private final Partitions partitions;
     private final ILazyAccessorReplacer replacer;
     protected final IParallelCacher cacher;
     private final IBulkOperationCallBack deleteCallBack;
 
     public ReplaceableCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager,
-            Set<Integer> partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) {
+            Partitions partitions, ILazyAccessorReplacer replacer, IParallelCacher cacher) {
         super(cloudClient, bucket, localIoManager);
         this.partitions = partitions;
         this.replacer = replacer;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
index 56eaeeb08b..68c0556fa3 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/lazy/accessor/SelectiveCloudAccessor.java
@@ -21,12 +21,12 @@ package org.apache.asterix.cloud.lazy.accessor;
 import static org.apache.asterix.cloud.util.CloudFileUtil.DATA_FILTER;
 
 import java.util.Collection;
-import java.util.Set;
 
 import org.apache.asterix.cloud.UncachedFileReference;
 import org.apache.asterix.cloud.clients.ICloudClient;
 import org.apache.asterix.cloud.lazy.IParallelCacher;
 import org.apache.asterix.cloud.lazy.filesystem.IHolePuncher;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 import org.apache.hyracks.api.io.IFileHandle;
@@ -37,7 +37,7 @@ public class SelectiveCloudAccessor extends ReplaceableCloudAccessor {
     private final IHolePuncher puncher;
 
     public SelectiveCloudAccessor(ICloudClient cloudClient, String bucket, IOManager localIoManager,
-            Set<Integer> partitions, IHolePuncher puncher, IParallelCacher cacher) {
+            Partitions partitions, IHolePuncher puncher, IParallelCacher cacher) {
         super(cloudClient, bucket, localIoManager, partitions, InitialCloudAccessor.NO_OP_REPLACER, cacher);
         this.puncher = puncher;
     }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
index 7befacf027..fd37e61c04 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/util/CloudFileUtil.java
@@ -49,7 +49,6 @@ public class CloudFileUtil {
         // First get the set of local files
         Set<FileReference> localFiles = ioManager.list(partitionPath);
         Iterator<FileReference> localFilesIter = localFiles.iterator();
-        LOGGER.info("Cleaning partition {}.", partitionPath.getRelativePath());
 
         // Reconcile local files and cloud files
         while (localFilesIter.hasNext()) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
index 45aaa49150..cc93b916cc 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -20,9 +20,13 @@ package org.apache.asterix.common.utils;
 
 import java.io.Serial;
 import java.io.Serializable;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.MutableBoolean;
 
 import it.unimi.dsi.fastutil.ints.IntIterator;
 import it.unimi.dsi.fastutil.ints.IntSortedSet;
+import it.unimi.dsi.fastutil.ints.IntSortedSets;
 
 /**
  * A specialized bit set for storing partition IDs.
@@ -36,21 +40,29 @@ public class Partitions implements Serializable {
     private static final long serialVersionUID = 1L;
     private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1;
     private static final int MIN_PARTITION_ID = -1;
+    private static final Partitions EMPTY = new Partitions(IntSortedSets.EMPTY_SET);
     private static final short MINUS_ONE = (short) -1;
     private final IntSortedSet delegate;
 
     public Partitions() {
-        delegate = new IntSortedBitSet();
+        this(new IntSortedBitSet());
     }
 
     public Partitions(int initialMaxValue) {
-        delegate = new IntSortedBitSet(initialMaxValue + 1);
+        this(new IntSortedBitSet(initialMaxValue + 1));
     }
 
     public Partitions(IntSortedSet delegate) {
         this.delegate = delegate;
     }
 
+    /**
+     * Returns an unmodifiable empty Partitions instance.
+     */
+    public static Partitions empty() {
+        return EMPTY;
+    }
+
     /**
      * Adds a partition ID to the set. Partition ID must be in the range [-1, 32766].
      * @param k the partition ID to add
@@ -72,6 +84,27 @@ public class Partitions implements Serializable {
         delegate.clear();
     }
 
+    public boolean addAll(Set<Integer> activePartitions) {
+        MutableBoolean retval = new MutableBoolean();
+        activePartitions.forEach(p -> {
+            if (add(p)) {
+                retval.setTrue();
+            }
+        });
+        return retval.booleanValue();
+    }
+
+    public int size() {
+        return delegate.size();
+    }
+
+    public boolean contains(int partitionNum) {
+        if (partitionNum > MAX_PARTITION_ID || partitionNum < MIN_PARTITION_ID) {
+            return false;
+        }
+        return delegate.contains((short) (partitionNum + 1));
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -107,4 +140,5 @@ public class Partitions implements Serializable {
         builder.append(']');
         return builder.toString();
     }
+
 }


(asterixdb) 23/25: [NO ISSUE][*DB][STO] Cloud, logging misc improvements

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit d86e2327d1c961c1bf6c666f6947fdab1560bc72
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Oct 19 20:43:09 2025 -0400

    [NO ISSUE][*DB][STO] Cloud, logging misc improvements
    
    Ext-ref: MB-69042,MB-68946
    Change-Id: I6b10c08e74e664f1b59ed0df3bf16b871360d66f
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20504
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Michael Blow <mb...@apache.org>
---
 .../asterix/api/http/server/StorageApiServlet.java |  11 +-
 .../app/message/StorageCleanupRequestMessage.java  |   4 +-
 .../apache/asterix/app/nc/NCAppRuntimeContext.java |   4 +-
 .../org/apache/asterix/app/nc/RecoveryManager.java |  34 +++--
 .../org/apache/asterix/app/nc/ReplicaManager.java  |  37 +++---
 .../nc/task/CloudToLocalStorageCachingTask.java    |  14 +-
 .../asterix/app/nc/task/LocalRecoveryTask.java     |  12 +-
 .../app/nc/task/LocalStorageCleanupTask.java       |   5 +-
 .../asterix/app/nc/task/UpdateNodeStatusTask.java  |   9 +-
 .../app/replication/NcLifecycleCoordinator.java    |  13 +-
 .../message/NCLifecycleTaskReportMessage.java      |  11 +-
 .../message/RegistrationTasksRequestMessage.java   |  12 +-
 .../message/RegistrationTasksResponseMessage.java  |   5 +-
 .../asterix/runtime/ClusterStateManagerTest.java   |   7 +-
 .../asterix/cloud/AbstractCloudIOManager.java      |   2 +-
 .../asterix/cloud/LocalPartitionBootstrapper.java  |   4 +-
 .../cloud/clients/aws/s3/S3ClientConfig.java       |  18 +--
 .../cloud/clients/aws/s3/S3ParallelDownloader.java |   2 +-
 .../org/apache/asterix/cloud/s3/LSMS3Test.java     |   2 +-
 .../common/cloud/IPartitionBootstrapper.java       |   4 +-
 .../common/cluster/IClusterStateManager.java       |   3 +-
 .../cluster/StorageComputePartitionsMap.java       |   7 +-
 .../asterix/common/config/CloudProperties.java     |  75 ++++++-----
 .../asterix/common/config/MetadataProperties.java  |   4 +-
 .../asterix/common/config/PropertiesAccessor.java  |   7 +-
 .../asterix/common/storage/IReplicaManager.java    |  10 +-
 .../common/transactions/IRecoveryManager.java      |   8 +-
 .../apache/asterix/common/utils/Partitions.java    | 146 +++++++++++++++++----
 .../replication/logging/RemoteLogsNotifier.java    |   7 +-
 .../replication/sync/ReplicaSynchronizer.java      |   4 +-
 .../asterix/runtime/utils/ClusterStateManager.java |  12 +-
 .../PersistentLocalResourceRepository.java         |  11 +-
 32 files changed, 305 insertions(+), 199 deletions(-)

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index 26fee28e10..b93938120b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -24,16 +24,15 @@ import java.io.IOException;
 import java.io.PrintWriter;
 import java.net.InetSocketAddress;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
+import java.util.function.IntPredicate;
 
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.replication.IPartitionReplica;
 import org.apache.asterix.common.storage.IReplicaManager;
 import org.apache.asterix.common.storage.ReplicaIdentifier;
 import org.apache.asterix.common.storage.ResourceStorageStats;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.http.api.IServletRequest;
@@ -121,11 +120,11 @@ public class StorageApiServlet extends AbstractServlet {
         return getStatus(partition::equals);
     }
 
-    private JsonNode getStatus(Predicate<Integer> predicate) {
+    private JsonNode getStatus(IntPredicate predicate) {
         final ArrayNode status = OBJECT_MAPPER.createArrayNode();
         final IReplicaManager storageSubsystem = appCtx.getReplicaManager();
-        final Set<Integer> partitions =
-                storageSubsystem.getPartitions().stream().filter(predicate).collect(Collectors.toSet());
+        final Partitions partitions = storageSubsystem.getPartitions().stream().filter(predicate)
+                .collect(Partitions::new, Partitions::add, Partitions::addAll);
         for (Integer partition : partitions) {
             final ObjectNode partitionJson = OBJECT_MAPPER.createObjectNode();
             partitionJson.put("partition", partition);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
index beaaac00fd..799510cbef 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/StorageCleanupRequestMessage.java
@@ -21,7 +21,6 @@ package org.apache.asterix.app.message;
 import static org.apache.hyracks.util.ExitUtil.EC_NC_FAILED_TO_NOTIFY_TASKS_COMPLETED;
 
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.api.INcApplicationContext;
@@ -30,6 +29,7 @@ import org.apache.asterix.common.messaging.CcIdentifiedMessage;
 import org.apache.asterix.common.messaging.api.INCMessageBroker;
 import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.common.IIndex;
@@ -57,7 +57,7 @@ public class StorageCleanupRequestMessage extends CcIdentifiedMessage implements
         INCMessageBroker broker = (INCMessageBroker) appContext.getServiceContext().getMessageBroker();
         PersistentLocalResourceRepository localResourceRepository =
                 (PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
-        Set<Integer> nodePartitions = appContext.getReplicaManager().getPartitions();
+        Partitions nodePartitions = appContext.getReplicaManager().getPartitions();
         Map<Long, LocalResource> localResources = localResourceRepository.getResources(lr -> true, nodePartitions);
         for (LocalResource resource : localResources.values()) {
             DatasetLocalResource lr = (DatasetLocalResource) resource.getResource();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index e0277aaf81..14d16515da 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -26,7 +26,6 @@ import java.rmi.server.UnicastRemoteObject;
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ExecutorService;
@@ -78,6 +77,7 @@ import org.apache.asterix.common.transactions.IRecoveryManager;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
 import org.apache.asterix.common.transactions.IRecoveryManagerFactory;
 import org.apache.asterix.common.transactions.ITransactionSubsystem;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.external.library.ExternalLibraryManager;
 import org.apache.asterix.file.StorageComponentProvider;
 import org.apache.asterix.metadata.MetadataManager;
@@ -296,7 +296,7 @@ public class NCAppRuntimeContext implements INcApplicationContext {
                         txnSubsystem.getLogManager(), virtualBufferCache, indexCheckpointManagerProvider, lockNotifier);
         localResourceRepository.setDatasetLifecycleManager(datasetLifecycleManager);
         final String nodeId = getServiceContext().getNodeId();
-        final Set<Integer> nodePartitions = metadataProperties.getNodePartitions(nodeId);
+        final Partitions nodePartitions = metadataProperties.getNodePartitions(nodeId);
         replicaManager = new ReplicaManager(this, nodePartitions);
         isShuttingdown = false;
         activeManager = new ActiveManager(threadExecutor, getServiceContext().getNodeId(),
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 269f6f7224..1b2c17d595 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -39,7 +39,6 @@ import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.api.INcApplicationContext;
@@ -61,6 +60,7 @@ import org.apache.asterix.common.transactions.ITransactionContext;
 import org.apache.asterix.common.transactions.ITransactionSubsystem;
 import org.apache.asterix.common.transactions.LogType;
 import org.apache.asterix.common.transactions.TxnId;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.asterix.transaction.management.service.logging.LogManager;
@@ -155,7 +155,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
     }
 
     @Override
-    public void startLocalRecovery(Set<Integer> partitions) throws IOException, ACIDException {
+    public void startLocalRecovery(Partitions partitions) throws IOException, ACIDException {
         state = SystemState.RECOVERING;
         LOGGER.info("starting recovery for partitions {}", partitions);
         Checkpoint systemCheckpoint = checkpointManager.getLatest();
@@ -175,7 +175,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         replayPartitionsLogs(partitions, logMgr.getLogReader(true), lowWaterMarkLSN, true);
     }
 
-    public synchronized void replayPartitionsLogs(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN,
+    public synchronized void replayPartitionsLogs(Partitions partitions, ILogReader logReader, long lowWaterMarkLSN,
             boolean closeOnFlushRedo) throws IOException, ACIDException {
         try {
             Set<Long> winnerJobSet = startRecoverysAnalysisPhase(partitions, logReader, lowWaterMarkLSN);
@@ -186,7 +186,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         }
     }
 
-    private synchronized Set<Long> startRecoverysAnalysisPhase(Set<Integer> partitions, ILogReader logReader,
+    private synchronized Set<Long> startRecoverysAnalysisPhase(Partitions partitions, ILogReader logReader,
             long lowWaterMarkLSN) throws IOException, ACIDException {
         int updateLogCount = 0;
         int entityCommitLogCount = 0;
@@ -274,8 +274,8 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         jobEntityWinners.add(logRecord);
     }
 
-    private synchronized void startRecoveryRedoPhase(Set<Integer> partitions, ILogReader logReader,
-            long lowWaterMarkLSN, Set<Long> winnerTxnSet, boolean closeOnFlushRedo) throws IOException, ACIDException {
+    private synchronized void startRecoveryRedoPhase(Partitions partitions, ILogReader logReader, long lowWaterMarkLSN,
+            Set<Long> winnerTxnSet, boolean closeOnFlushRedo) throws IOException, ACIDException {
         int redoCount = 0;
         long txnId = 0;
 
@@ -491,21 +491,20 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
 
     private long getRemoteMinFirstLSN() throws HyracksDataException {
         // find the min first lsn of partitions that are replicated on this node
-        final Set<Integer> allPartitions = localResourceRepository.getAllPartitions();
-        final Set<Integer> masterPartitions = appCtx.getReplicaManager().getPartitions();
+        final Partitions allPartitions = localResourceRepository.getAllPartitions();
+        final Partitions masterPartitions = appCtx.getReplicaManager().getPartitions();
         allPartitions.removeAll(masterPartitions);
         return getPartitionsMinLSN(allPartitions);
     }
 
-    private long getPartitionsMinLSN(Set<Integer> partitions) throws HyracksDataException {
+    private long getPartitionsMinLSN(Partitions partitions) throws HyracksDataException {
         final IIndexCheckpointManagerProvider idxCheckpointMgrProvider = appCtx.getIndexCheckpointManagerProvider();
         long minRemoteLSN = Long.MAX_VALUE;
         for (Integer partition : partitions) {
             final List<DatasetResourceReference> partitionResources = localResourceRepository.getResources(resource -> {
                 DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource();
                 return dsResource.getPartition() == partition;
-            }, Collections.singleton(partition)).values().stream().map(DatasetResourceReference::of)
-                    .collect(Collectors.toList());
+            }, Partitions.singleton(partition)).values().stream().map(DatasetResourceReference::of).toList();
             for (DatasetResourceReference indexRef : partitionResources) {
                 try {
                     final IIndexCheckpointManager idxCheckpointMgr = idxCheckpointMgrProvider.get(indexRef);
@@ -524,15 +523,14 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
     }
 
     @Override
-    public long getPartitionsMaxLSN(Set<Integer> partitions) throws HyracksDataException {
+    public long getPartitionsMaxLSN(Partitions partitions) throws HyracksDataException {
         final IIndexCheckpointManagerProvider idxCheckpointMgrProvider = appCtx.getIndexCheckpointManagerProvider();
         long maxLSN = 0;
-        for (Integer partition : partitions) {
+        for (int partition : partitions) {
             final List<DatasetResourceReference> partitionResources = localResourceRepository.getResources(resource -> {
                 DatasetLocalResource dsResource = (DatasetLocalResource) resource.getResource();
                 return dsResource.getPartition() == partition;
-            }, Collections.singleton(partition)).values().stream().map(DatasetResourceReference::of)
-                    .collect(Collectors.toList());
+            }, Partitions.singleton(partition)).values().stream().map(DatasetResourceReference::of).toList();
             for (DatasetResourceReference indexRef : partitionResources) {
                 try {
                     final IIndexCheckpointManager idxCheckpointMgr = idxCheckpointMgrProvider.get(indexRef);
@@ -549,7 +547,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
     }
 
     @Override
-    public synchronized void replayReplicaPartitionLogs(Set<Integer> partitions, boolean flush)
+    public synchronized void replayReplicaPartitionLogs(Partitions partitions, boolean flush)
             throws HyracksDataException {
         //replay logs > minLSN that belong to these partitions
         try {
@@ -597,7 +595,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         FileUtils.deleteQuietly(recoveryFolderPath.toFile());
     }
 
-    protected void cleanUp(Set<Integer> partitions) throws HyracksDataException {
+    protected void cleanUp(Partitions partitions) throws HyracksDataException {
         // the cleanup is currently done by PersistentLocalResourceRepository#clean
     }
 
@@ -657,7 +655,7 @@ public class RecoveryManager implements IRecoveryManager, ILifeCycleComponent {
         TxnEntityId loserEntity;
         List<Long> undoLSNSet = null;
         //get active partitions on this node
-        Set<Integer> activePartitions = appCtx.getReplicaManager().getPartitions();
+        Partitions activePartitions = appCtx.getReplicaManager().getPartitions();
         ILogReader logReader = logMgr.getLogReader(false);
         try {
             logReader.setPosition(firstLSN);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 101b1b0fe2..a1eee08225 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -19,12 +19,9 @@
 package org.apache.asterix.app.nc;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -34,6 +31,7 @@ import org.apache.asterix.common.replication.IPartitionReplica;
 import org.apache.asterix.common.storage.IReplicaManager;
 import org.apache.asterix.common.storage.ReplicaIdentifier;
 import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.replication.api.PartitionReplica;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.client.NodeStatus;
@@ -44,6 +42,9 @@ import org.apache.hyracks.util.annotations.ThreadSafe;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
+import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
+
 @ThreadSafe
 public class ReplicaManager implements IReplicaManager {
     private static final Logger LOGGER = LogManager.getLogger();
@@ -52,17 +53,17 @@ public class ReplicaManager implements IReplicaManager {
     /**
      * the partitions to which the current node is master
      */
-    private final Map<Integer, Object> partitions = new HashMap<>();
+    private final Int2ObjectMap<PartitionSyncLock> partitions = new Int2ObjectOpenHashMap<>();
     /**
      * current replicas
      */
     private final Map<ReplicaIdentifier, PartitionReplica> replicas = new HashMap<>();
-    private final Set<Integer> nodeOriginatedPartitions = new HashSet<>();
+    private final Partitions nodeOriginatedPartitions = new Partitions();
 
-    public ReplicaManager(INcApplicationContext appCtx, Set<Integer> partitions) {
+    public ReplicaManager(INcApplicationContext appCtx, Partitions partitions) {
         this.appCtx = appCtx;
-        for (Integer partition : partitions) {
-            this.partitions.put(partition, new Object());
+        for (int partition : partitions) {
+            this.partitions.put(partition, new PartitionSyncLock());
         }
         setNodeOriginatedPartitions(appCtx);
     }
@@ -109,15 +110,15 @@ public class ReplicaManager implements IReplicaManager {
     }
 
     @Override
-    public synchronized Set<Integer> getPartitions() {
-        return Collections.unmodifiableSet(partitions.keySet());
+    public synchronized Partitions getPartitions() {
+        return new Partitions(partitions.keySet()).unmodifiable();
     }
 
     @Override
-    public synchronized void setActivePartitions(Set<Integer> activePartitions) {
+    public synchronized void setActivePartitions(Partitions activePartitions) {
         partitions.clear();
-        for (Integer partition : activePartitions) {
-            partitions.put(partition, new Object());
+        for (int partition : activePartitions) {
+            partitions.put(partition, new PartitionSyncLock());
         }
     }
 
@@ -132,9 +133,9 @@ public class ReplicaManager implements IReplicaManager {
         if (!appCtx.isCloudDeployment()) {
             localResourceRepository.cleanup(partition);
             final IRecoveryManager recoveryManager = appCtx.getTransactionSubsystem().getRecoveryManager();
-            recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()), true);
+            recoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Partitions.collector()), true);
         }
-        partitions.put(partition, new Object());
+        partitions.put(partition, new PartitionSyncLock());
     }
 
     @Override
@@ -151,8 +152,8 @@ public class ReplicaManager implements IReplicaManager {
     }
 
     @Override
-    public synchronized Object getPartitionSyncLock(int partition) {
-        Object syncLock = partitions.get(partition);
+    public synchronized PartitionSyncLock getPartitionSyncLock(int partition) {
+        PartitionSyncLock syncLock = partitions.get(partition);
         if (syncLock == null) {
             throw new IllegalStateException("partition " + partition + " is not active on this node");
         }
@@ -187,7 +188,7 @@ public class ReplicaManager implements IReplicaManager {
     }
 
     private void setNodeOriginatedPartitions(INcApplicationContext appCtx) {
-        Set<Integer> nodePartitions =
+        Partitions nodePartitions =
                 appCtx.getMetadataProperties().getNodePartitions(appCtx.getServiceContext().getNodeId());
         nodeOriginatedPartitions.addAll(nodePartitions);
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
index cc594884ba..00205a64c6 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/CloudToLocalStorageCachingTask.java
@@ -18,13 +18,11 @@
  */
 package org.apache.asterix.app.nc.task;
 
-import java.util.Arrays;
-import java.util.Set;
-
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.cloud.IPartitionBootstrapper;
 import org.apache.asterix.common.transactions.Checkpoint;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -37,13 +35,12 @@ public class CloudToLocalStorageCachingTask implements INCLifecycleTask {
 
     private static final Logger LOGGER = LogManager.getLogger();
 
-    private static final long serialVersionUID = 1L;
-    private final Set<Integer> storagePartitions;
+    private static final long serialVersionUID = 2L;
+    private final Partitions storagePartitions;
     private final boolean metadataNode;
     private final int metadataPartitionId;
 
-    public CloudToLocalStorageCachingTask(Set<Integer> storagePartitions, boolean metadataNode,
-            int metadataPartitionId) {
+    public CloudToLocalStorageCachingTask(Partitions storagePartitions, boolean metadataNode, int metadataPartitionId) {
         this.storagePartitions = storagePartitions;
         this.metadataNode = metadataNode;
         this.metadataPartitionId = metadataPartitionId;
@@ -78,7 +75,6 @@ public class CloudToLocalStorageCachingTask implements INCLifecycleTask {
 
     @Override
     public String toString() {
-        return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : "
-                + Arrays.toString(storagePartitions.toArray()) + " }";
+        return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : " + storagePartitions + " }";
     }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalRecoveryTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalRecoveryTask.java
index b8addb29bc..e906de77f0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalRecoveryTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalRecoveryTask.java
@@ -19,22 +19,21 @@
 package org.apache.asterix.app.nc.task;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.Set;
 
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.service.IControllerService;
 
 public class LocalRecoveryTask implements INCLifecycleTask {
 
-    private static final long serialVersionUID = 1L;
-    private final Set<Integer> partitions;
+    private static final long serialVersionUID = 2L;
+    private final Partitions partitions;
 
-    public LocalRecoveryTask(Set<Integer> partitions) {
+    public LocalRecoveryTask(Partitions partitions) {
         this.partitions = partitions;
     }
 
@@ -52,7 +51,6 @@ public class LocalRecoveryTask implements INCLifecycleTask {
 
     @Override
     public String toString() {
-        return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : "
-                + Arrays.toString(partitions.toArray()) + " }";
+        return "{ \"class\" : \"" + getClass().getSimpleName() + "\", \"partitions\" : " + partitions + " }";
     }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
index 61a07d15ab..673e401cf5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/LocalStorageCleanupTask.java
@@ -18,12 +18,11 @@
  */
 package org.apache.asterix.app.nc.task;
 
-import java.util.Set;
-
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.dataflow.DatasetLocalResource;
 import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -45,7 +44,7 @@ public class LocalStorageCleanupTask implements INCLifecycleTask {
                 (PersistentLocalResourceRepository) appContext.getLocalResourceRepository();
         localResourceRepository.deleteCorruptedResources();
         deleteInvalidMetadataIndexes(localResourceRepository);
-        final Set<Integer> nodePartitions = appContext.getReplicaManager().getPartitions();
+        final Partitions nodePartitions = appContext.getReplicaManager().getPartitions();
         INcApplicationContext appCtx = (INcApplicationContext) cs.getApplicationContext();
         if (appCtx.isCloudDeployment() && nodePartitions.contains(metadataPartitionId)) {
             appCtx.getTransactionSubsystem().getTransactionManager().rollbackMetadataTransactionsWithoutWAL();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
index 17eff4ad81..b4b2d0987a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/UpdateNodeStatusTask.java
@@ -18,11 +18,10 @@
  */
 package org.apache.asterix.app.nc.task;
 
-import java.util.Set;
-
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.storage.IReplicaManager;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.client.NodeStatus;
 import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -34,11 +33,11 @@ import org.apache.logging.log4j.Logger;
 public class UpdateNodeStatusTask implements INCLifecycleTask {
 
     private static final Logger LOGGER = LogManager.getLogger();
-    private static final long serialVersionUID = 2L;
+    private static final long serialVersionUID = 3L;
     private final NodeStatus status;
-    private final Set<Integer> activePartitions;
+    private final Partitions activePartitions;
 
-    public UpdateNodeStatusTask(NodeStatus status, Set<Integer> activePartitions) {
+    public UpdateNodeStatusTask(NodeStatus status, Partitions activePartitions) {
         this.status = status;
         this.activePartitions = activePartitions;
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
index fff1bc118c..9ccb66a0b7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/NcLifecycleCoordinator.java
@@ -60,6 +60,7 @@ import org.apache.asterix.common.messaging.api.ICCMessageBroker;
 import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.asterix.common.replication.INcLifecycleCoordinator;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.metadata.MetadataManager;
 import org.apache.asterix.replication.messaging.ReplicaFailedMessage;
 import org.apache.http.client.utils.URIBuilder;
@@ -172,7 +173,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
     }
 
     protected List<INCLifecycleTask> buildNCRegTasks(String nodeId, NodeStatus nodeStatus, SystemState state,
-            Set<Integer> activePartitions) {
+            Partitions activePartitions) {
         LOGGER.info(
                 "Building registration tasks for node {} with status {} and system state: {} and active partitions {}",
                 nodeId, nodeStatus, state, activePartitions);
@@ -217,9 +218,9 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
     }
 
     protected List<INCLifecycleTask> buildIdleNcRegTasks(String newNodeId, boolean metadataNode, SystemState state,
-            Set<Integer> activePartitions) {
+            Partitions activePartitions) {
         final List<INCLifecycleTask> tasks = new ArrayList<>();
-        Set<Integer> nodeActivePartitions = getNodeActivePartitions(newNodeId, activePartitions, metadataNode, state);
+        Partitions nodeActivePartitions = getNodeActivePartitions(newNodeId, activePartitions, metadataNode, state);
         tasks.add(new UpdateNodeStatusTask(NodeStatus.BOOTING, nodeActivePartitions));
         int metadataPartitionId = clusterManager.getMetadataPartition().getPartitionId();
         // Add any cloud-related tasks
@@ -256,7 +257,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
         return tasks;
     }
 
-    protected void addCloudTasks(List<INCLifecycleTask> tasks, Set<Integer> computePartitions, boolean metadataNode,
+    protected void addCloudTasks(List<INCLifecycleTask> tasks, Partitions computePartitions, boolean metadataNode,
             int metadataPartitionId) {
         IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext();
         if (!appCtx.isCloudDeployment()) {
@@ -265,7 +266,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
 
         StorageComputePartitionsMap map = clusterManager.getStorageComputeMap();
         map = map == null ? StorageComputePartitionsMap.computePartitionsMap(clusterManager) : map;
-        Set<Integer> storagePartitions = map.getStoragePartitions(computePartitions);
+        Partitions storagePartitions = map.getStoragePartitions(computePartitions);
         tasks.add(new CloudToLocalStorageCachingTask(storagePartitions, metadataNode, metadataPartitionId));
     }
 
@@ -322,7 +323,7 @@ public class NcLifecycleCoordinator implements INcLifecycleCoordinator {
         return true;
     }
 
-    protected Set<Integer> getNodeActivePartitions(String nodeId, Set<Integer> nodePartitions, boolean metadataNode,
+    protected Partitions getNodeActivePartitions(String nodeId, Partitions nodePartitions, boolean metadataNode,
             SystemState state) {
         if (metadataNode) {
             nodePartitions.add(clusterManager.getMetadataPartition().getPartitionId());
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
index b2a2dd771a..082f5b5462 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/NCLifecycleTaskReportMessage.java
@@ -18,25 +18,24 @@
  */
 package org.apache.asterix.app.replication.message;
 
-import java.util.Set;
-
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
 import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.asterix.common.utils.NcLocalCounters;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
 public class NCLifecycleTaskReportMessage implements INCLifecycleMessage, ICcAddressedMessage {
 
-    private static final long serialVersionUID = 1L;
+    private static final long serialVersionUID = 2L;
     private final String nodeId;
     private final boolean success;
     private Throwable exception;
     private final NcLocalCounters localCounters;
-    private final Set<Integer> activePartitions;
+    private final Partitions activePartitions;
 
     public NCLifecycleTaskReportMessage(String nodeId, boolean success, NcLocalCounters localCounters,
-            Set<Integer> activePartitions) {
+            Partitions activePartitions) {
         this.nodeId = nodeId;
         this.success = success;
         this.localCounters = localCounters;
@@ -73,7 +72,7 @@ public class NCLifecycleTaskReportMessage implements INCLifecycleMessage, ICcAdd
         return MessageType.REGISTRATION_TASKS_RESULT;
     }
 
-    public Set<Integer> getActivePartitions() {
+    public Partitions getActivePartitions() {
         return activePartitions;
     }
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
index 9e95ac644d..976308c81d 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksRequestMessage.java
@@ -19,13 +19,13 @@
 package org.apache.asterix.app.replication.message;
 
 import java.util.Map;
-import java.util.Set;
 
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
 import org.apache.asterix.common.messaging.api.INCMessageBroker;
 import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.client.NodeStatus;
 import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -37,15 +37,15 @@ import org.apache.logging.log4j.Logger;
 public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICcAddressedMessage {
 
     private static final Logger LOGGER = LogManager.getLogger();
-    private static final long serialVersionUID = 2L;
+    private static final long serialVersionUID = 3L;
     private final SystemState state;
     private final String nodeId;
     private final NodeStatus nodeStatus;
     private final Map<String, Object> secrets;
-    private final Set<Integer> activePartitions;
+    private final Partitions activePartitions;
 
     public RegistrationTasksRequestMessage(String nodeId, NodeStatus nodeStatus, SystemState state,
-            Map<String, Object> secretsEphemeral, Set<Integer> activePartitions) {
+            Map<String, Object> secretsEphemeral, Partitions activePartitions) {
         this.state = state;
         this.nodeId = nodeId;
         this.nodeStatus = nodeStatus;
@@ -54,7 +54,7 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc
     }
 
     public static void send(CcId ccId, NodeControllerService cs, NodeStatus nodeStatus, SystemState systemState,
-            Map<String, Object> secretsEphemeral, Set<Integer> activePartitions) throws HyracksDataException {
+            Map<String, Object> secretsEphemeral, Partitions activePartitions) throws HyracksDataException {
         try {
             RegistrationTasksRequestMessage msg = new RegistrationTasksRequestMessage(cs.getId(), nodeStatus,
                     systemState, secretsEphemeral, activePartitions);
@@ -92,7 +92,7 @@ public class RegistrationTasksRequestMessage implements INCLifecycleMessage, ICc
         return secrets;
     }
 
-    public Set<Integer> getActivePartitions() {
+    public Partitions getActivePartitions() {
         return activePartitions;
     }
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
index 9d378f5a41..285563e645 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/RegistrationTasksResponseMessage.java
@@ -18,9 +18,7 @@
  */
 package org.apache.asterix.app.replication.message;
 
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.asterix.common.api.INCLifecycleTask;
 import org.apache.asterix.common.api.INcApplicationContext;
@@ -29,6 +27,7 @@ import org.apache.asterix.common.messaging.api.INCMessageBroker;
 import org.apache.asterix.common.messaging.api.INcAddressedMessage;
 import org.apache.asterix.common.replication.INCLifecycleMessage;
 import org.apache.asterix.common.utils.NcLocalCounters;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.service.IControllerService;
 import org.apache.hyracks.control.nc.NodeControllerService;
@@ -74,7 +73,7 @@ public class RegistrationTasksResponseMessage extends CcIdentifiedMessage
                 LOGGER.debug("returning local counters to cc: {}", localCounter);
             }
             // wrap the returned partitions in a hash set to make it serializable
-            Set<Integer> nodeActivePartitions = new HashSet<>(appCtx.getReplicaManager().getPartitions());
+            Partitions nodeActivePartitions = new Partitions(appCtx.getReplicaManager().getPartitions());
             NCLifecycleTaskReportMessage result =
                     new NCLifecycleTaskReportMessage(nodeId, success, localCounter, nodeActivePartitions);
             result.setException(exception);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
index 53b9294e2b..6b99253791 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ClusterStateManagerTest.java
@@ -19,9 +19,7 @@
 package org.apache.asterix.runtime;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
@@ -37,6 +35,7 @@ import org.apache.asterix.common.config.StorageProperties;
 import org.apache.asterix.common.metadata.IMetadataBootstrap;
 import org.apache.asterix.common.utils.NcLocalCounters;
 import org.apache.asterix.common.utils.PartitioningScheme;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.hyracks.bootstrap.CCApplication;
 import org.apache.asterix.runtime.transaction.ResourceIdManager;
 import org.apache.asterix.runtime.utils.BulkTxnIdFactory;
@@ -277,8 +276,8 @@ public class ClusterStateManagerTest {
         return localCounters;
     }
 
-    private static Set<Integer> getNodeActivePartitions(String nodeId) {
-        Set<Integer> activePartitions = new HashSet<>();
+    private static Partitions getNodeActivePartitions(String nodeId) {
+        Partitions activePartitions = new Partitions();
         switch (nodeId) {
             case NC1:
                 activePartitions.add(0);
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
index 7712f08653..6fc271e382 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/AbstractCloudIOManager.java
@@ -126,7 +126,7 @@ public abstract class AbstractCloudIOManager extends IOManager implements IParti
     }
 
     @Override
-    public final void bootstrap(Set<Integer> activePartitions, List<FileReference> currentOnDiskPartitions,
+    public final void bootstrap(Partitions activePartitions, List<FileReference> currentOnDiskPartitions,
             boolean metadataNode, int metadataPartition, boolean ensureCompleteBootstrap) throws HyracksDataException {
         partitions.clear();
         partitions.addAll(activePartitions);
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LocalPartitionBootstrapper.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LocalPartitionBootstrapper.java
index f2451ee04f..d5d0d87f3d 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LocalPartitionBootstrapper.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/LocalPartitionBootstrapper.java
@@ -20,10 +20,10 @@
 package org.apache.asterix.cloud;
 
 import java.util.List;
-import java.util.Set;
 
 import org.apache.asterix.common.cloud.IPartitionBootstrapper;
 import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
@@ -48,7 +48,7 @@ public class LocalPartitionBootstrapper implements IPartitionBootstrapper {
     }
 
     @Override
-    public void bootstrap(Set<Integer> activePartitions, List<FileReference> currentOnDiskPartitions,
+    public void bootstrap(Partitions activePartitions, List<FileReference> currentOnDiskPartitions,
             boolean metadataNode, int metadataPartition, boolean ensureCompleteBootstrap) throws HyracksDataException {
         for (FileReference onDiskPartition : currentOnDiskPartitions) {
             int partitionNum = StoragePathUtil.getPartitionNumFromRelativePath(onDiskPartition.getAbsolutePath());
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
index f05a0e5820..b6683ad20c 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ClientConfig.java
@@ -47,18 +47,19 @@ public final class S3ClientConfig {
     private final boolean forcePathStyle;
     private final boolean disableSslVerify;
     private final boolean storageListEventuallyConsistent;
+    private final boolean enableCrtClient;
 
     public S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
-            long profilerLogInterval, int writeBufferSize) {
+            long profilerLogInterval, int writeBufferSize, boolean enableCrtClient) {
         this(region, endpoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, 1, 0, 0, 0, false, false,
-                false, 0, 0);
+                false, 0, 0, enableCrtClient);
     }
 
     private S3ClientConfig(String region, String endpoint, String prefix, boolean anonymousAuth,
             long profilerLogInterval, int writeBufferSize, long tokenAcquireTimeout, int writeMaxRequestsPerSeconds,
             int readMaxRequestsPerSeconds, int requestsMaxHttpConnections, boolean forcePathStyle,
             boolean disableSslVerify, boolean storageListEventuallyConsistent, int requestsMaxPendingHttpConnections,
-            int requestsHttpConnectionAcquireTimeout) {
+            int requestsHttpConnectionAcquireTimeout, boolean enableCrtClient) {
         this.region = Objects.requireNonNull(region, "region");
         this.endpoint = endpoint;
         this.prefix = Objects.requireNonNull(prefix, "prefix");
@@ -74,6 +75,7 @@ public final class S3ClientConfig {
         this.forcePathStyle = forcePathStyle;
         this.disableSslVerify = disableSslVerify;
         this.storageListEventuallyConsistent = storageListEventuallyConsistent;
+        this.enableCrtClient = enableCrtClient;
     }
 
     public static S3ClientConfig of(CloudProperties cloudProperties) {
@@ -85,7 +87,7 @@ public final class S3ClientConfig {
                 cloudProperties.isStorageForcePathStyle(), cloudProperties.isStorageDisableSSLVerify(),
                 cloudProperties.isStorageListEventuallyConsistent(),
                 cloudProperties.getRequestsMaxPendingHttpConnections(),
-                cloudProperties.getRequestsHttpConnectionAcquireTimeout());
+                cloudProperties.getRequestsHttpConnectionAcquireTimeout(), cloudProperties.isS3EnableCrtClient());
     }
 
     public static S3ClientConfig of(Map<String, String> configuration, int writeBufferSize) {
@@ -99,7 +101,7 @@ public final class S3ClientConfig {
         String prefix = "";
         boolean anonymousAuth = false;
 
-        return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize);
+        return new S3ClientConfig(region, endPoint, prefix, anonymousAuth, profilerLogInterval, writeBufferSize, false);
     }
 
     public String getRegion() {
@@ -116,7 +118,7 @@ public final class S3ClientConfig {
 
     public boolean isLocalS3Provider() {
         // to workaround https://github.com/findify/s3mock/issues/187 in our S3Mock, we encode/decode keys
-        return isS3Mock();
+        return false; //isS3Mock();
     }
 
     public AwsCredentialsProvider createCredentialsProvider() {
@@ -167,8 +169,8 @@ public final class S3ClientConfig {
         return storageListEventuallyConsistent;
     }
 
-    private boolean isS3Mock() {
-        return endpoint != null && !endpoint.isEmpty();
+    public boolean isCrtClientEnabled() {
+        return enableCrtClient;
     }
 
 }
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
index 2acc12e7ba..50d05c66d5 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/aws/s3/S3ParallelDownloader.java
@@ -181,7 +181,7 @@ class S3ParallelDownloader implements IParallelDownloader {
 
     private static S3AsyncClient createAsyncClient(S3ClientConfig config) {
         // CRT client is not supported by all local S3 providers, but provides a better performance with AWS S3
-        if (!config.isLocalS3Provider()) {
+        if (config.isCrtClientEnabled()) {
             return createS3CrtAsyncClient(config);
         }
         return createS3AsyncClient(config);
diff --git a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
index fad8081d99..1bb0f746e4 100644
--- a/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
+++ b/asterixdb/asterix-cloud/src/test/java/org/apache/asterix/cloud/s3/LSMS3Test.java
@@ -68,7 +68,7 @@ public class LSMS3Test extends AbstractLSMTest {
         LOGGER.info("Client created successfully");
         int writeBufferSize = StorageUtil.getIntSizeInBytes(5, StorageUtil.StorageUnit.MEGABYTE);
         S3ClientConfig config =
-                new S3ClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", true, 0, writeBufferSize);
+                new S3ClientConfig(MOCK_SERVER_REGION, MOCK_SERVER_HOSTNAME, "", true, 0, writeBufferSize, false);
         CLOUD_CLIENT = new S3CloudClient(config, ICloudGuardian.NoOpCloudGuardian.INSTANCE);
     }
 
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cloud/IPartitionBootstrapper.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cloud/IPartitionBootstrapper.java
index 88e88abac4..3cca8f5d25 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cloud/IPartitionBootstrapper.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cloud/IPartitionBootstrapper.java
@@ -19,9 +19,9 @@
 package org.apache.asterix.common.cloud;
 
 import java.util.List;
-import java.util.Set;
 
 import org.apache.asterix.common.transactions.IRecoveryManager;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.FileReference;
 
@@ -52,6 +52,6 @@ public interface IPartitionBootstrapper {
      * @param metadataPartition       metadata partition number
      * @param ensureCompleteBootstrap ensures the metadata catalog was fully bootstrapped
      */
-    void bootstrap(Set<Integer> activePartitions, List<FileReference> currentOnDiskPartitions, boolean metadataNode,
+    void bootstrap(Partitions activePartitions, List<FileReference> currentOnDiskPartitions, boolean metadataNode,
             int metadataPartition, boolean ensureCompleteBootstrap) throws HyracksDataException;
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 6c0982a38e..ffc554c33e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -27,6 +27,7 @@ import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.utils.NcLocalCounters;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.config.IOption;
@@ -65,7 +66,7 @@ public interface IClusterStateManager {
      * @param activePartitions
      * @throws HyracksDataException
      */
-    void updateNodeState(String nodeId, boolean active, NcLocalCounters ncLocalCounters, Set<Integer> activePartitions)
+    void updateNodeState(String nodeId, boolean active, NcLocalCounters ncLocalCounters, Partitions activePartitions)
             throws HyracksDataException;
 
     /**
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
index dc62cac667..ef440eb678 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
@@ -21,13 +21,14 @@ package org.apache.asterix.common.cluster;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
+import org.apache.asterix.common.utils.Partitions;
+
 public class StorageComputePartitionsMap {
 
     private final Map<Integer, ComputePartition> storageComputeMap = new HashMap<>();
@@ -83,8 +84,8 @@ public class StorageComputePartitionsMap {
      * @param computePartitions the current active compute partitions
      * @return computePartitions's corresponding storage partitions
      */
-    public Set<Integer> getStoragePartitions(Set<Integer> computePartitions) {
-        Set<Integer> storagePartitions = new HashSet<>();
+    public Partitions getStoragePartitions(Partitions computePartitions) {
+        Partitions storagePartitions = new Partitions();
         for (Map.Entry<Integer, ComputePartition> entry : storageComputeMap.entrySet()) {
             ComputePartition computePartition = entry.getValue();
             if (computePartitions.contains(computePartition.getId())) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
index f556fcbb94..392282f6ec 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CloudProperties.java
@@ -28,8 +28,10 @@ import static org.apache.hyracks.control.common.config.OptionTypes.getRangedInte
 import static org.apache.hyracks.util.StorageUtil.StorageUnit.GIGABYTE;
 
 import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
 
 import org.apache.asterix.common.cloud.CloudCachePolicy;
+import org.apache.hyracks.api.config.IApplicationConfig;
 import org.apache.hyracks.api.config.IOption;
 import org.apache.hyracks.api.config.IOptionType;
 import org.apache.hyracks.api.config.Section;
@@ -70,7 +72,11 @@ public class CloudProperties extends AbstractProperties {
         CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT(POSITIVE_INTEGER, 120),
         CLOUD_STORAGE_FORCE_PATH_STYLE(BOOLEAN, false),
         CLOUD_STORAGE_DISABLE_SSL_VERIFY(BOOLEAN, false),
-        CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT(BOOLEAN, false);
+        CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT(BOOLEAN, false),
+        CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT(BOOLEAN, (Function<IApplicationConfig, Boolean>) app -> {
+            String endpoint = app.getString(CLOUD_STORAGE_ENDPOINT);
+            return endpoint == null || endpoint.isEmpty();
+        });
 
         private final IOptionType interpreter;
         private final Object defaultValue;
@@ -80,6 +86,11 @@ public class CloudProperties extends AbstractProperties {
             this.defaultValue = defaultValue;
         }
 
+        <T> Option(IOptionType<T> interpreter, Function<IApplicationConfig, T> defaultOption) {
+            this.interpreter = interpreter;
+            this.defaultValue = defaultOption;
+        }
+
         @Override
         public Section section() {
             switch (this) {
@@ -134,62 +145,55 @@ public class CloudProperties extends AbstractProperties {
                             + "all partitions upon booting, whereas 'lazy' caching will download a file upon"
                             + " request to open it. 'selective' caching will act as the 'lazy' policy; however, "
                             + " it allows to use the local disk(s) as a cache, where pages and indexes can be "
-                            + " cached or evicted according to the pressure imposed on the local disks."
-                            + " (default: 'selective')";
+                            + " cached or evicted according to the pressure imposed on the local disks";
                 case CLOUD_STORAGE_ALLOCATION_PERCENTAGE:
                     return "The percentage of the total disk space that should be allocated for data storage when the"
                             + " 'selective' caching policy is used. The remaining will act as a buffer for "
-                            + " query workspace (i.e., for query operations that require spilling to disk)."
-                            + " (default: 80% of the total disk space)";
+                            + " query workspace (i.e., for query operations that require spilling to disk)";
                 case CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE:
                     return "The percentage of the used storage space at which the disk sweeper starts freeing space by"
                             + " punching holes in stored indexes or by evicting them entirely, "
-                            + " when the 'selective' caching policy is used."
-                            + " (default: 90% of the allocated space for storage)";
+                            + " when the 'selective' caching policy is used";
                 case CLOUD_STORAGE_DISK_MONITOR_INTERVAL:
                     return "The disk monitoring interval time (in seconds): determines how often the system"
-                            + " checks for pressure on disk space when using the 'selective' caching policy."
-                            + " (default : 120 seconds)";
+                            + " checks for pressure on disk space when using the 'selective' caching policy";
                 case CLOUD_STORAGE_INDEX_INACTIVE_DURATION_THRESHOLD:
-                    return "The duration in minutes to consider an index is inactive. (default: 360 or 6 hours)";
+                    return "The duration in minutes to consider an index is inactive";
                 case CLOUD_STORAGE_DEBUG_MODE_ENABLED:
-                    return "Whether or not the debug mode is enabled when using the 'selective' caching policy."
-                            + "(default: false)";
+                    return "Whether or not the debug mode is enabled when using the 'selective' caching policy";
                 case CLOUD_STORAGE_DEBUG_SWEEP_THRESHOLD_SIZE:
                     return "For debugging only. Pressure size will be the current used space + the additional bytes"
                             + " provided by this configuration option instead of using "
-                            + " CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE."
-                            + " (default: 0. I.e., CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE will be used by default)";
+                            + " CLOUD_STORAGE_SWEEP_THRESHOLD_PERCENTAGE";
                 case CLOUD_PROFILER_LOG_INTERVAL:
-                    return "The waiting time (in minutes) to log cloud request statistics. The minimum is 1 minute."
-                            + " Note: by default, the logging is disabled. Enabling it could perturb the performance of cloud requests";
+                    return "The waiting time (in minutes) to log cloud request statistics. The minimum is 1 minute"
+                            + " Note: enabling this logging may perturb the performance of cloud requests";
                 case CLOUD_ACQUIRE_TOKEN_TIMEOUT:
                     return "The waiting time (in milliseconds) if a requesting thread failed to acquire a token if the"
-                            + " rate limit of cloud requests exceeded (default: 100, min: 1, and max: 5000)";
+                            + " rate limit of cloud requests exceeded (min: 1, max: 5000)";
                 case CLOUD_MAX_WRITE_REQUESTS_PER_SECOND:
-                    return "The maximum number of write requests per second (default: 250, 0 means unlimited)";
+                    return "The maximum number of write requests per second (0 means unlimited)";
                 case CLOUD_MAX_READ_REQUESTS_PER_SECOND:
-                    return "The maximum number of read requests per second (default: 1500, 0 means unlimited)";
+                    return "The maximum number of read requests per second (0 means unlimited)";
                 case CLOUD_WRITE_BUFFER_SIZE:
-                    return "The write buffer size in bytes. (default: 8MB, min: 5MB)";
+                    return "The write buffer size in bytes. (min: 5MiB)";
                 case CLOUD_EVICTION_PLAN_REEVALUATE_THRESHOLD:
-                    return "The number of cloud reads for re-evaluating an eviction plan. (default: 50)";
+                    return "The number of cloud reads for re-evaluating an eviction plan";
                 case CLOUD_REQUESTS_MAX_HTTP_CONNECTIONS:
-                    return "The maximum number of HTTP connections to use concurrently for cloud requests per node. (default: 1000)";
+                    return "The maximum number of HTTP connections to use concurrently for cloud requests per node";
                 case CLOUD_REQUESTS_MAX_PENDING_HTTP_CONNECTIONS:
-                    return "The maximum number of HTTP connections allowed to wait for a connection per node. (default: 10000)";
+                    return "The maximum number of HTTP connections allowed to wait for a connection per node";
                 case CLOUD_REQUESTS_HTTP_CONNECTION_ACQUIRE_TIMEOUT:
-                    return "The waiting time (in seconds) to acquire an HTTP connection before failing the request."
-                            + " (default: 120 seconds)";
+                    return "The waiting time (in seconds) to acquire an HTTP connection before failing the request";
                 case CLOUD_STORAGE_FORCE_PATH_STYLE:
-                    return "Indicates whether or not to force path style when accessing the cloud storage. (default:"
-                            + " false)";
+                    return "Indicates whether or not to force path style when accessing the cloud storage";
                 case CLOUD_STORAGE_DISABLE_SSL_VERIFY:
-                    return "Indicates whether or not to disable SSL certificate verification on the cloud storage. "
-                            + "(default: false)";
+                    return "Indicates whether or not to disable SSL certificate verification on the cloud storage";
                 case CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT:
                     return "Indicates whether or not deleted objects may be contained in list operations for some time"
-                            + "after they are deleted. (default: false)";
+                            + "after they are deleted";
+                case CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT:
+                    return "Indicates whether or not to use the AWS CRT S3 client for async requests";
                 default:
                     throw new IllegalStateException("NYI: " + this);
             }
@@ -205,6 +209,13 @@ public class CloudProperties extends AbstractProperties {
             return defaultValue;
         }
 
+        @Override
+        public String usageDefaultOverride(IApplicationConfig accessor, Function<IOption, String> optionPrinter) {
+            if (this == CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT) {
+                return "true when no custom endpoint is set, otherwise false";
+            }
+            return IOption.super.usageDefaultOverride(accessor, optionPrinter);
+        }
     }
 
     public String getStorageScheme() {
@@ -309,4 +320,8 @@ public class CloudProperties extends AbstractProperties {
     public boolean isStorageListEventuallyConsistent() {
         return accessor.getBoolean(Option.CLOUD_STORAGE_LIST_EVENTUALLY_CONSISTENT);
     }
+
+    public boolean isS3EnableCrtClient() {
+        return accessor.getBoolean(Option.CLOUD_STORAGE_S3_ENABLE_CRT_CLIENT);
+    }
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
index 8d18bfd8e9..568802c059 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/MetadataProperties.java
@@ -24,10 +24,10 @@ import static org.apache.hyracks.control.common.config.OptionTypes.STRING;
 
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.SortedMap;
 
 import org.apache.asterix.common.cluster.ClusterPartition;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.config.IApplicationConfig;
 import org.apache.hyracks.api.config.IOption;
 import org.apache.hyracks.api.config.IOptionType;
@@ -124,7 +124,7 @@ public class MetadataProperties extends AbstractProperties {
         return accessor.getClusterPartitions();
     }
 
-    public Set<Integer> getNodePartitions(String nodeId) {
+    public Partitions getNodePartitions(String nodeId) {
         return accessor.getNodePartitions(nodeId);
     }
 
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
index 80f9a17f99..84e2ff00fe 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesAccessor.java
@@ -36,11 +36,11 @@ import java.util.SortedMap;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import org.apache.asterix.common.cluster.ClusterPartition;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.PrintUtil;
 import org.apache.commons.lang3.mutable.MutableInt;
 import org.apache.hyracks.algebricks.common.utils.Pair;
@@ -196,9 +196,10 @@ public class PropertiesAccessor implements IApplicationConfig {
         return clusterPartitions;
     }
 
-    public Set<Integer> getNodePartitions(String nodeId) {
+    public Partitions getNodePartitions(String nodeId) {
         ClusterPartition[] nodeClusterPartitions = nodePartitionsMap.get(nodeId);
-        return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId).collect(Collectors.toSet());
+        return Arrays.stream(nodeClusterPartitions).map(ClusterPartition::getPartitionId)
+                .collect(Partitions.collector());
     }
 
     public List<AsterixExtension> getExtensions() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
index 14fc9d8e0c..daedf8774e 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
@@ -19,9 +19,9 @@
 package org.apache.asterix.common.storage;
 
 import java.util.List;
-import java.util.Set;
 
 import org.apache.asterix.common.replication.IPartitionReplica;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 
 public interface IReplicaManager {
@@ -54,14 +54,14 @@ public interface IReplicaManager {
      *
      * @return The list of partition
      */
-    Set<Integer> getPartitions();
+    Partitions getPartitions();
 
     /**
      * Sets the node active partitions
      *
      * @param activePartitions
      */
-    void setActivePartitions(Set<Integer> activePartitions);
+    void setActivePartitions(Partitions activePartitions);
 
     /**
      * Promotes a partition by making this node its master replica
@@ -87,7 +87,7 @@ public interface IReplicaManager {
      *
      * @return the synchronization lock
      */
-    Object getPartitionSyncLock(int partition);
+    PartitionSyncLock getPartitionSyncLock(int partition);
 
     /**
      * Gets the partition replicas matching {@code id}
@@ -110,4 +110,6 @@ public interface IReplicaManager {
      * @return true if the partition is originated by this node, otherwise false.
      */
     boolean isPartitionOrigin(int partition);
+
+    record PartitionSyncLock() {}
 }
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
index 3437d4202d..cf96816ee9 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/transactions/IRecoveryManager.java
@@ -21,9 +21,9 @@ package org.apache.asterix.common.transactions;
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
-import java.util.Set;
 
 import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 
@@ -109,7 +109,7 @@ public interface IRecoveryManager {
      * @throws IOException
      * @throws ACIDException
      */
-    void startLocalRecovery(Set<Integer> partitions) throws IOException, ACIDException;
+    void startLocalRecovery(Partitions partitions) throws IOException, ACIDException;
 
     /**
      * Gets the LSN used in {@code partitions} or 0 if no LSNs are found
@@ -118,7 +118,7 @@ public interface IRecoveryManager {
      * @return the maximum used LSN
      * @throws HyracksDataException
      */
-    long getPartitionsMaxLSN(Set<Integer> partitions) throws HyracksDataException;
+    long getPartitionsMaxLSN(Partitions partitions) throws HyracksDataException;
 
     /**
      * Replay the commited transactions' logs belonging to {@code partitions}. if {@code flush} is true,
@@ -128,7 +128,7 @@ public interface IRecoveryManager {
      * @param flush
      * @throws HyracksDataException
      */
-    void replayReplicaPartitionLogs(Set<Integer> partitions, boolean flush) throws HyracksDataException;
+    void replayReplicaPartitionLogs(Partitions partitions, boolean flush) throws HyracksDataException;
 
     /**
      * Ensures that {@code datasetPartitionIndexes} are consistent by performing component id level recovery
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
index e939ce13b7..1405661a7b 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/Partitions.java
@@ -21,38 +21,58 @@ package org.apache.asterix.common.utils;
 import java.io.Serial;
 import java.io.Serializable;
 import java.util.Set;
+import java.util.stream.Collector;
+import java.util.stream.IntStream;
 
 import org.apache.commons.lang3.mutable.MutableBoolean;
 
+import it.unimi.dsi.fastutil.ints.IntIterable;
 import it.unimi.dsi.fastutil.ints.IntIterator;
+import it.unimi.dsi.fastutil.ints.IntSet;
 import it.unimi.dsi.fastutil.ints.IntSortedSet;
 import it.unimi.dsi.fastutil.ints.IntSortedSets;
 
 /**
- * A specialized bit set for storing partition IDs.
- * Partition IDs are in the range [-1, 32766] (inclusive).
- * The value -1 is used to represent the "unpartitioned" partition.
- * The value 32767 is reserved for the internal shift and cannot be used as a partition ID.
- * This class internally shifts the partition IDs by +1 to fit into an unsigned short range [0, 32767].
+ * A specialized data structure for efficiently storing partitions.
+ * Partitions are in the range [-1, 2_147_483_646] (inclusive).
+ * The value -1 is used to represent the "metadata" partition.
+ * The value 2_147_483_647 is reserved for the internal shift and cannot be used as a partition.
+ * This class internally shifts the partitions by +1 to fit into the signed int range [0, 2_147_483_647].
  */
-public class Partitions implements Serializable {
+public class Partitions implements IntIterable, Serializable {
     @Serial
     private static final long serialVersionUID = 1L;
-    private static final int MAX_PARTITION_ID = Integer.MAX_VALUE - 1;
-    private static final int MIN_PARTITION_ID = -1;
+    private static final int DELTA = -1;
+    private static final int MAX_PARTITION = Integer.MAX_VALUE + DELTA;
+    private static final int MIN_PARTITION = DELTA;
+    private static final int DEFAULT_MAX_PARTITIONS = 128;
     private static final Partitions EMPTY = new Partitions(IntSortedSets.EMPTY_SET);
-    private static final short MINUS_ONE = (short) -1;
     private final IntSortedSet delegate;
 
     public Partitions() {
-        this(new IntSortedBitSet());
+        this(DEFAULT_MAX_PARTITIONS);
     }
 
-    public Partitions(int initialMaxValue) {
-        this(new IntSortedBitSet(initialMaxValue + 1));
+    public Partitions(int initialMaxPartition) {
+        this(new IntSortedBitSet(encodePartition(initialMaxPartition)));
     }
 
-    public Partitions(IntSortedSet delegate) {
+    public Partitions(Set<Integer> initialValues) {
+        this();
+        initialValues.forEach(this::add);
+    }
+
+    public Partitions(IntSet initialValues) {
+        this();
+        initialValues.forEach(this::add);
+    }
+
+    public Partitions(Partitions initialValues) {
+        this();
+        addAll(initialValues);
+    }
+
+    private Partitions(IntSortedSet delegate) {
         this.delegate = delegate;
     }
 
@@ -63,17 +83,43 @@ public class Partitions implements Serializable {
         return EMPTY;
     }
 
+    public static Partitions singleton(int partition) {
+        return new Partitions(IntSortedSets.singleton(encodePartition(partition)));
+    }
+
+    public static Collector<? super Integer, Partitions, Partitions> collector() {
+        return Collector.of(Partitions::new, Partitions::add, (l, r) -> {
+            l.addAll(r);
+            return l;
+        }, Collector.Characteristics.UNORDERED, Collector.Characteristics.IDENTITY_FINISH);
+    }
+
     /**
-     * Adds a partition ID to the set. Partition ID must be in the range [-1, 32766].
-     * @param k the partition ID to add
-     * @return true if the partition ID was added, false if it was already present
-     * @throws IllegalArgumentException if the partition ID is out of range
+     * Adds a partition to the set. Partition must be in the range [-1, 2_147_483_646].
+     * @param partition the partition to add
+     * @return true if the partition was added, false if it was already present
+     * @throws IllegalArgumentException if the partition is out of range
      */
-    public boolean add(int k) {
-        if (k > MAX_PARTITION_ID || k < MIN_PARTITION_ID) {
-            throw new IllegalArgumentException("Partition number " + k + " out of range");
+    public boolean add(int partition) {
+        checkRange(partition);
+        return delegate.add(encodePartition(partition));
+    }
+
+    /**
+     * Removes a partition from the set. Partition must be in the range [-1, 2_147_483_646].
+     * @param partition the partition to remove
+     * @return true if the partition was added, false if it was already present
+     * @throws IllegalArgumentException if the partition is out of range
+     */
+    public boolean remove(int partition) {
+        checkRange(partition);
+        return delegate.remove(encodePartition(partition));
+    }
+
+    private static void checkRange(int partition) {
+        if (partition > MAX_PARTITION || partition < MIN_PARTITION) {
+            throw new IllegalArgumentException("Partition number " + partition + " out of range");
         }
-        return delegate.add((short) (k + 1));
     }
 
     public boolean isEmpty() {
@@ -94,15 +140,44 @@ public class Partitions implements Serializable {
         return retval.booleanValue();
     }
 
+    public boolean addAll(Partitions activePartitions) {
+        return delegate.addAll(activePartitions.delegate);
+    }
+
     public int size() {
         return delegate.size();
     }
 
-    public boolean contains(int partitionNum) {
-        if (partitionNum > MAX_PARTITION_ID || partitionNum < MIN_PARTITION_ID) {
+    public boolean contains(int partition) {
+        if (partition > MAX_PARTITION || partition < MIN_PARTITION) {
             return false;
         }
-        return delegate.contains((short) (partitionNum + 1));
+        return delegate.contains(encodePartition(partition));
+    }
+
+    @Override
+    public IntIterator iterator() {
+        return new IntIterator() {
+            private final IntIterator delegateIter = delegate.iterator();
+
+            @Override
+            public boolean hasNext() {
+                return delegateIter.hasNext();
+            }
+
+            @Override
+            public int nextInt() {
+                return decodePartition(delegateIter.nextInt());
+            }
+        };
+    }
+
+    public int getMinPartition() {
+        return decodePartition(delegate.getFirst());
+    }
+
+    public int getMaxPartition() {
+        return decodePartition(delegate.getLast());
     }
 
     @Override
@@ -130,17 +205,36 @@ public class Partitions implements Serializable {
         StringBuilder builder = new StringBuilder();
         builder.append('[');
         if (delegate.firstInt() == 0) {
-            builder.append(MINUS_ONE);
+            builder.append(DELTA);
             iter.nextInt(); // this is the zero we just printed
             if (iter.hasNext()) {
                 builder.append(',');
             }
         }
         if (iter.hasNext()) {
-            IntUtil.appendCompact(iter, builder, MINUS_ONE);
+            IntUtil.appendCompact(iter, builder, DELTA);
         }
         builder.append(']');
         return builder.toString();
     }
 
+    public void removeAll(Partitions masterPartitions) {
+        delegate.removeAll(masterPartitions.delegate);
+    }
+
+    public Partitions unmodifiable() {
+        return new Partitions(IntSortedSets.unmodifiable(delegate));
+    }
+
+    public IntStream stream() {
+        return delegate.intStream().map(Partitions::decodePartition);
+    }
+
+    private static int encodePartition(int partition) {
+        return partition - DELTA;
+    }
+
+    private static int decodePartition(int value) {
+        return value + DELTA;
+    }
 }
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
index 42a3c0b212..13bd31b3c0 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/logging/RemoteLogsNotifier.java
@@ -19,12 +19,10 @@
 package org.apache.asterix.replication.logging;
 
 import java.io.IOException;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.dataflow.DatasetLocalResource;
@@ -32,6 +30,7 @@ import org.apache.asterix.common.storage.DatasetResourceReference;
 import org.apache.asterix.common.storage.IIndexCheckpointManager;
 import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
 import org.apache.asterix.common.transactions.LogType;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.replication.messaging.ReplicationProtocol;
 import org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -90,9 +89,9 @@ class RemoteLogsNotifier implements Runnable {
             return dls.getDatasetId() == datasetId && dls.getPartition() == resourcePartition;
         };
         final Map<Long, LocalResource> resources =
-                localResourceRep.getResources(replicaIndexesPredicate, Collections.singleton(resourcePartition));
+                localResourceRep.getResources(replicaIndexesPredicate, Partitions.singleton(resourcePartition));
         final List<DatasetResourceReference> replicaIndexesRef =
-                resources.values().stream().map(DatasetResourceReference::of).collect(Collectors.toList());
+                resources.values().stream().map(DatasetResourceReference::of).toList();
         for (DatasetResourceReference replicaIndexRef : replicaIndexesRef) {
             final IIndexCheckpointManager indexCheckpointManager = indexCheckpointManagerProvider.get(replicaIndexRef);
             synchronized (indexCheckpointManager) {
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaSynchronizer.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaSynchronizer.java
index 68ccd54e50..b0a8c54a47 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaSynchronizer.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/sync/ReplicaSynchronizer.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 
 import org.apache.asterix.common.api.INcApplicationContext;
 import org.apache.asterix.common.replication.IReplicationStrategy;
+import org.apache.asterix.common.storage.IReplicaManager.PartitionSyncLock;
 import org.apache.asterix.common.transactions.ICheckpointManager;
 import org.apache.asterix.replication.api.PartitionReplica;
 import org.apache.asterix.replication.messaging.CheckpointPartitionIndexesTask;
@@ -48,7 +49,8 @@ public class ReplicaSynchronizer {
 
     public void sync(boolean register, boolean deltaRecovery) throws IOException {
         LOGGER.debug("starting replica sync process for replica {}", replica);
-        Object partitionLock = appCtx.getReplicaManager().getPartitionSyncLock(replica.getIdentifier().getPartition());
+        PartitionSyncLock partitionLock =
+                appCtx.getReplicaManager().getPartitionSyncLock(replica.getIdentifier().getPartition());
         synchronized (partitionLock) {
             LOGGER.trace("acquired partition replica lock");
             final ICheckpointManager checkpointManager = appCtx.getTransactionSubsystem().getCheckpointManager();
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 668decfa3d..cb97929807 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -30,7 +30,6 @@ import java.util.SortedMap;
 import java.util.TreeSet;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Predicate;
-import java.util.stream.Collectors;
 
 import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
 import org.apache.asterix.common.cluster.ClusterPartition;
@@ -43,6 +42,7 @@ import org.apache.asterix.common.replication.INcLifecycleCoordinator;
 import org.apache.asterix.common.transactions.IResourceIdManager;
 import org.apache.asterix.common.utils.NcLocalCounters;
 import org.apache.asterix.common.utils.PartitioningScheme;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StorageConstants;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -152,15 +152,15 @@ public class ClusterStateManager implements IClusterStateManager {
 
     @Override
     public synchronized void updateNodeState(String nodeId, boolean active, NcLocalCounters localCounters,
-            Set<Integer> activePartitions) {
+            Partitions activePartitions) {
         if (active) {
             updateClusterCounters(nodeId, localCounters);
             participantNodes.add(nodeId);
             if (appCtx.isCloudDeployment()) {
                 // node compute partitions never change
                 ClusterPartition[] nodePartitions = getNodePartitions(nodeId);
-                activePartitions =
-                        Arrays.stream(nodePartitions).map(ClusterPartition::getPartitionId).collect(Collectors.toSet());
+                activePartitions = Arrays.stream(nodePartitions).map(ClusterPartition::getPartitionId)
+                        .collect(Partitions.collector());
                 activateNodePartitions(nodeId, activePartitions);
             } else {
                 activateNodePartitions(nodeId, activePartitions);
@@ -548,8 +548,8 @@ public class ClusterStateManager implements IClusterStateManager {
         });
     }
 
-    private synchronized void activateNodePartitions(String nodeId, Set<Integer> activePartitions) {
-        for (Integer partitionId : activePartitions) {
+    private synchronized void activateNodePartitions(String nodeId, Partitions activePartitions) {
+        for (int partitionId : activePartitions) {
             updateClusterPartition(partitionId, nodeId, true);
         }
     }
diff --git a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index cb4e068aab..d53c258a76 100644
--- a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -56,6 +56,7 @@ import org.apache.asterix.common.storage.IIndexCheckpointManager;
 import org.apache.asterix.common.storage.IIndexCheckpointManagerProvider;
 import org.apache.asterix.common.storage.ResourceReference;
 import org.apache.asterix.common.storage.ResourceStorageStats;
+import org.apache.asterix.common.utils.Partitions;
 import org.apache.asterix.common.utils.StorageConstants;
 import org.apache.asterix.common.utils.StoragePathUtil;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -288,7 +289,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
         }
     }
 
-    public Map<Long, LocalResource> getResources(Predicate<LocalResource> filter, Set<Integer> partitions)
+    public Map<Long, LocalResource> getResources(Predicate<LocalResource> filter, Partitions partitions)
             throws HyracksDataException {
         beforeReadAccess();
         try {
@@ -427,12 +428,12 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
         }
     }
 
-    public Set<Integer> getAllPartitions() throws HyracksDataException {
+    public Partitions getAllPartitions() throws HyracksDataException {
         beforeReadAccess();
         try {
             return loadAndGetAllResources().values().stream().map(LocalResource::getResource)
                     .map(DatasetLocalResource.class::cast).map(DatasetLocalResource::getPartition)
-                    .collect(Collectors.toSet());
+                    .collect(Partitions.collector());
         } finally {
             afterReadAccess();
         }
@@ -479,7 +480,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
     public Map<Long, LocalResource> getPartitionResources(int partition) throws HyracksDataException {
         beforeReadAccess();
         try {
-            return getResources(r -> true, Collections.singleton(partition));
+            return getResources(r -> true, Partitions.singleton(partition));
         } finally {
             afterReadAccess();
         }
@@ -712,7 +713,7 @@ public class PersistentLocalResourceRepository implements ILocalResourceReposito
         return null;
     }
 
-    public long getDatasetSize(DatasetCopyIdentifier datasetIdentifier, Set<Integer> nodePartitions)
+    public long getDatasetSize(DatasetCopyIdentifier datasetIdentifier, Partitions nodePartitions)
             throws HyracksDataException {
         beforeReadAccess();
         try {


(asterixdb) 08/25: [NO ISSUE][EXT]: Use ABFSS instead of deprecated WASBS

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 48509e02c4097a938628e7f5f13115a9d32103c8
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Mon Oct 6 23:05:41 2025 +0300

    [NO ISSUE][EXT]: Use ABFSS instead of deprecated WASBS
    
    Details:
    - Avoid importing old azure-storage as hadoop 3.3.4
      is compatible with Azure SDK v12.
    
    Ext-ref: MB-68683
    Change-Id: Id2ed162f300f1549296fcdfc0e87b65e61421a6b
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20466
    Reviewed-by: Michael Blow <mb...@apache.org>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../external/util/azure/blob_storage/AzureConstants.java |  2 +-
 asterixdb/pom.xml                                        | 16 +++++-----------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
index 9ade27baeb..675981f7b4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureConstants.java
@@ -60,6 +60,6 @@ public class AzureConstants {
     public static final String HADOOP_AZURE_FS_ACCOUNT_KEY = "fs.azure.account.key";
     //Used when a connectionString is provided
     public static final String HADOOP_AZURE_FS_SAS = "fs.azure.sas";
-    public static final String HADOOP_AZURE_BLOB_PROTOCOL = "wasbs";
+    public static final String HADOOP_AZURE_BLOB_PROTOCOL = "abfss";
     public static final String HADOOP_AZURE_DATALAKE_PROTOCOL = "abfss";
 }
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 05f0f37958..909b4551e5 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1799,22 +1799,16 @@
       <!-- Hadoop AWS end -->
       <!-- Hadoop Azure start -->
       <dependency>
-        <!-- Pick a newer Azure connector -->
-        <groupId>com.microsoft.azure</groupId>
-        <artifactId>azure-storage</artifactId>
-        <version>${hadoop-azuresdk.version}</version>
+        <groupId>org.apache.hadoop</groupId>
+        <artifactId>hadoop-azure</artifactId>
+        <version>${hadoop.version}</version>
         <exclusions>
           <exclusion>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
+            <groupId>com.microsoft.azure</groupId>
+            <artifactId>azure-storage</artifactId>
           </exclusion>
         </exclusions>
       </dependency>
-      <dependency>
-        <groupId>org.apache.hadoop</groupId>
-        <artifactId>hadoop-azure</artifactId>
-        <version>${hadoop.version}</version>
-      </dependency>
       <!-- Hadoop Azure end -->
       <!-- Hadoop GCS start -->
       <dependency>


(asterixdb) 03/25: [NO ISSUE][*DB][EXT] Make IDataParser extend Closeable, misc

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 4874143e7d7328b1a220c542b45cd45b8a8135a9
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Oct 3 16:10:27 2025 -0400

    [NO ISSUE][*DB][EXT] Make IDataParser extend Closeable, misc
    
    Ext-ref: MB-68827
    Change-Id: Id8d97f948d048e5c204bb0b78c089420b76dddc9
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20443
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Tested-by: Michael Blow <mb...@apache.org>
---
 .../java/org/apache/asterix/active/ActiveManager.java  |  1 +
 .../org/apache/asterix/external/api/IDataParser.java   | 13 +++++++++----
 .../apache/asterix/external/api/IRecordConverter.java  |  9 +++++++--
 .../dataflow/FeedRecordDataFlowController.java         |  2 ++
 .../data/std/util/ByteArrayAccessibleOutputStream.java | 18 ++++++++++++++++++
 .../org/apache/hyracks/http/server/utils/HttpUtil.java |  6 ++++--
 .../common/dataflow/IndexDropOperatorNodePushable.java |  2 +-
 .../storage/am/lsm/common/impls/LSMHarness.java        | 16 ++++++++++++++++
 8 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
index a75ee33979..0a3ce800fb 100644
--- a/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
+++ b/asterixdb/asterix-active/src/main/java/org/apache/asterix/active/ActiveManager.java
@@ -192,6 +192,7 @@ public class ActiveManager {
                     + " this node before the cluster controller sent the stop request");
         } else {
             executor.execute(() -> {
+                Thread.currentThread().setName(runtimeId.toString());
                 try {
                     stopIfRunning(runtime, content.getTimeout(), content.getUnit());
                 } catch (Throwable th) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
index 5dbc383feb..d3a744831d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IDataParser.java
@@ -18,6 +18,7 @@
  */
 package org.apache.asterix.external.api;
 
+import java.io.Closeable;
 import java.io.DataOutput;
 
 import org.apache.asterix.builders.IARecordBuilder;
@@ -37,14 +38,18 @@ import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
 
-public interface IDataParser {
+public interface IDataParser extends Closeable {
+    @Override
+    default void close() {
+        // Default no-op
+    }
 
     /*
      * The following two static methods are expensive. right now, they are used by RSSFeeds and
      * Twitter feed
      * TODO: Get rid of them
      */
-    public static void writeRecord(AMutableRecord record, DataOutput dataOutput, IARecordBuilder recordBuilder)
+    static void writeRecord(AMutableRecord record, DataOutput dataOutput, IARecordBuilder recordBuilder)
             throws HyracksDataException {
         ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
         int numFields = record.getType().getFieldNames().length;
@@ -58,7 +63,7 @@ public interface IDataParser {
     }
 
     @SuppressWarnings("unchecked")
-    public static void writeObject(IAObject obj, DataOutput dataOutput) throws HyracksDataException {
+    static void writeObject(IAObject obj, DataOutput dataOutput) throws HyracksDataException {
         switch (obj.getType().getTypeTag()) {
             case OBJECT: {
                 IARecordBuilder recordBuilder = new RecordBuilder();
@@ -105,7 +110,7 @@ public interface IDataParser {
         }
     }
 
-    public static <T> void toBytes(T serializable, ArrayBackedValueStorage buffer, ISerializerDeserializer<T> serde)
+    static <T> void toBytes(T serializable, ArrayBackedValueStorage buffer, ISerializerDeserializer<T> serde)
             throws HyracksDataException {
         buffer.reset();
         DataOutput out = buffer.getDataOutput();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
index f544ca041b..abb41f2a0f 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
@@ -18,13 +18,14 @@
  */
 package org.apache.asterix.external.api;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.function.LongSupplier;
 
 @FunctionalInterface
-public interface IRecordConverter<I, O> {
+public interface IRecordConverter<I, O> extends Closeable {
 
-    public O convert(IRawRecord<? extends I> input) throws IOException;
+    O convert(IRawRecord<? extends I> input) throws IOException;
 
     /**
      * Configures the converter with information suppliers from the {@link IRecordReader} data source.
@@ -33,4 +34,8 @@ public interface IRecordConverter<I, O> {
      */
     default void configure(LongSupplier lineNumber) {
     }
+
+    default void close() {
+        // default no-op
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
index 4279ebd712..6ef54974b3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/dataflow/FeedRecordDataFlowController.java
@@ -252,6 +252,8 @@ public class FeedRecordDataFlowController<T> extends AbstractFeedDataFlowControl
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 throw HyracksDataException.create(e);
+            } finally {
+                dataParser.close();
             }
             return true;
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
index 1a806cd1fd..17f9a2e649 100644
--- a/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
+++ b/hyracks-fullstack/hyracks/hyracks-data/hyracks-data-std/src/main/java/org/apache/hyracks/data/std/util/ByteArrayAccessibleOutputStream.java
@@ -125,4 +125,22 @@ public class ByteArrayAccessibleOutputStream extends ByteArrayOutputStream {
         ensureCapacity(bytesRequired);
         count = bytesRequired;
     }
+
+    /**
+     * Reset the stream and shrink the internal buffer if its size is larger than newSize.
+     * @param newSize
+     * @return true if the internal buffer was reallocated, false otherwise.
+     */
+    public boolean resetAndShrink(int newSize) {
+        reset();
+        if (buf.length > newSize) {
+            buf = new byte[newSize];
+            return true;
+        }
+        return false;
+    }
+
+    public int capacity() {
+        return buf.length;
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
index 9f01123c0b..cac8719194 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/utils/HttpUtil.java
@@ -240,7 +240,8 @@ public class HttpUtil {
                 try {
                     response.close();
                 } catch (IOException e) {
-                    LOGGER.debug("{} ignoring exception thrown on stream close due to interrupt", description, e);
+                    LOGGER.debug("{} ignoring exception thrown on stream close due to interrupt: {}", description,
+                            String.valueOf(e));
                 }
             });
             try {
@@ -249,7 +250,8 @@ public class HttpUtil {
                 LOGGER.warn("{} did not exit on stream close due to interrupt after 1s", description);
                 readFuture.cancel(true);
             } catch (ExecutionException ee) {
-                LOGGER.debug("ignoring exception awaiting aborted {} shutdown", description, ee);
+                LOGGER.debug("ignoring exception awaiting aborted {} shutdown: {}", description,
+                        String.valueOf(ee.getCause()));
             }
             throw ex;
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
index 3b6669ef52..5b9053b108 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexDropOperatorNodePushable.java
@@ -84,7 +84,7 @@ public class IndexDropOperatorNodePushable extends AbstractOperatorNodePushable
                 return;
             } catch (HyracksDataException e) {
                 if (isIgnorable(e)) {
-                    LOGGER.debug("Ignoring exception on drop", e);
+                    LOGGER.debug("Ignoring exception on drop: {}", String.valueOf(e));
                     return;
                 }
                 if (canRetry(e)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 017e767721..2ac29fe640 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -19,6 +19,9 @@
 
 package org.apache.hyracks.storage.am.lsm.common.impls;
 
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,6 +66,8 @@ import org.apache.logging.log4j.Logger;
 
 public class LSMHarness implements ILSMHarness {
     private static final Logger LOGGER = LogManager.getLogger();
+    private static final DateTimeFormatter OP_THREAD_TIMESTAMP =
+            DateTimeFormatter.ofPattern("HH:mm:ss").withZone(ZoneId.systemDefault());
 
     protected final ILSMIndex lsmIndex;
     protected final ILSMIOOperationScheduler ioScheduler;
@@ -543,7 +548,9 @@ public class LSMHarness implements ILSMHarness {
     }
 
     public void doIo(ILSMIOOperation operation) {
+        String origName = Thread.currentThread().getName();
         try {
+            Thread.currentThread().setName(threadName(operation));
             operation.getCallback().beforeOperation(operation);
             ILSMDiskComponent newComponent = operation.getIOOpertionType() == LSMIOOperationType.FLUSH
                     ? lsmIndex.flush(operation) : lsmIndex.merge(operation);
@@ -569,6 +576,7 @@ public class LSMHarness implements ILSMHarness {
                             lsmIndex, th);
                 }
             }
+            Thread.currentThread().setName(origName);
         }
         // if the operation failed, we need to cleanup files
         if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
@@ -576,6 +584,14 @@ public class LSMHarness implements ILSMHarness {
         }
     }
 
+    private static String threadName(ILSMIOOperation operation) {
+        if (operation.getIOOpertionType() == LSMIOOperationType.NOOP) {
+            return String.valueOf(operation.getIOOpertionType());
+        }
+        return operation.getIOOpertionType() + ":" + operation.getTarget().getRelativePath() + "@"
+                + OP_THREAD_TIMESTAMP.format(ZonedDateTime.now());
+    }
+
     @Override
     public void merge(ILSMIOOperation operation) throws HyracksDataException {
         if (LOGGER.isDebugEnabled()) {


(asterixdb) 15/25: [ASTERIXDB-3392][EXT] Fix Hdfs copy to parquet test failures

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 0c58b3585417809e5bb49f0e26b431367a7eafef
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Fri Oct 17 14:19:50 2025 +0530

    [ASTERIXDB-3392][EXT] Fix Hdfs copy to parquet test failures
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Ext-ref: MB-68610
    
    Change-Id: Ied1de68d5e668b8aac6c24c650d78f9e5ac367c6
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20494
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Preetham Poluparthi <pr...@apache.org>
---
 .../asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml    | 1 -
 1 file changed, 1 deletion(-)

diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
index 34b3b24f18..89395ac4b5 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp_hdfs.xml
@@ -255,7 +255,6 @@
         <expected-error>ASX1205: Invalid Parquet Writer Version provided '3'. Supported values: [1, 2]</expected-error>
         <expected-error>ASX0039: Expected integer value, got yvghc (in line 22, at column 6)</expected-error>
         <expected-error>ASX1209: Maximum value allowed for 'max-schemas' is 10. Found 15</expected-error>
-        <expected-error>HYR0133: Schema could not be inferred, empty types found in the result</expected-error>
         <expected-error>HYR0134: Schema Limit exceeded, maximum number of heterogeneous schemas allowed : '2'</expected-error>
         <expected-error>ASX1204: 'rectangle' type not supported in parquet format</expected-error>
         <expected-error>ASX0072: Parquet does not support arrays containing mixed data types</expected-error>


(asterixdb) 10/25: Merge branch 'gerrit/neo' into 'gerrit/trinity'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit e27f6f95ad4fb1718c87d7578595aed54331f71d
Merge: 48509e02c4 b20e3c66d9
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Oct 8 16:13:47 2025 -0400

    Merge branch 'gerrit/neo' into 'gerrit/trinity'
    
     * [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page... (MB-68818)
    
    Ext-ref: MB-68818
    Change-Id: I716c724053126aebeb71e3182367d45df91ccb0b

 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 +++++++++++++++++++++
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 2 files changed, 103 insertions(+), 3 deletions(-)



(asterixdb) 06/25: [NO ISSUE][EXT]: Add method to generate an external ID

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 9e12be5e57e1d18b31bbc0093aaf00d3956829a4
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Mon Oct 6 18:17:15 2025 +0300

    [NO ISSUE][EXT]: Add method to generate an external ID
    
    Ext-ref: MB-68761
    Change-Id: I1bd710782c231bceddd3c0a1e2f9600c3d625f66
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20465
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../java/org/apache/asterix/external/util/aws/s3/S3Utils.java  | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index d8dd478da7..7783456871 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -26,6 +26,7 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.UUID;
 import java.util.function.BiPredicate;
 import java.util.regex.Matcher;
 
@@ -298,4 +299,13 @@ public class S3Utils {
                 + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
                 + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
     }
+
+    /**
+     * Generates a random external ID to be used in cross-account role assumption.
+     *
+     * @return external id
+     */
+    public static String generateExternalId() {
+        return UUID.randomUUID().toString();
+    }
 }


(asterixdb) 04/25: Merge branch 'gerrit/ionic' into 'gerrit/phoenix'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 74652bf3f3afa6314a063b4e433936961dc19eb8
Merge: 95d55701d4 75dd626ac1
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Oct 3 14:00:14 2025 -0400

    Merge branch 'gerrit/ionic' into 'gerrit/phoenix'
    
     * [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
    
    Ext-ref: MB-68818
    Change-Id: I0e6a28ded7237e9222b972fb8ad43681d6e48897

 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 +++++++++++++++++++++
 .../lsm/flush/FlushColumnTupleWriter.java          |  14 +--
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 3 files changed, 111 insertions(+), 9 deletions(-)


(asterixdb) 14/25: Merge branch 'gerrit/trinity' into 'gerrit/phoenix'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 95f77257b45373d2bfc65d85d41559305283d3cb
Merge: 384570e475 e27f6f95ad
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Oct 15 19:58:53 2025 -0400

    Merge branch 'gerrit/trinity' into 'gerrit/phoenix'
    
     * [NO ISSUE][*DB][EXT] Make IDataParser extend Closeable, misc
     * [NO ISSUE][EXT]: Use ABFSS instead of deprecated WASBS
     * [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
    
    Ext-ref: MB-67819
    Change-Id: I49f69c95163d54b4c199f234291047fd01c27775

 .../org/apache/asterix/active/ActiveManager.java   |  1 +
 .../apache/asterix/external/api/IDataParser.java   | 13 ++++++---
 .../asterix/external/api/IRecordConverter.java     |  8 +++++-
 .../dataflow/FeedRecordDataFlowController.java     |  2 ++
 .../util/azure/blob_storage/AzureConstants.java    |  2 +-
 asterixdb/pom.xml                                  | 16 ++++-------
 .../std/util/ByteArrayAccessibleOutputStream.java  | 18 +++++++++++++
 .../apache/hyracks/http/server/utils/HttpUtil.java |  6 +++--
 .../dataflow/IndexDropOperatorNodePushable.java    |  2 +-
 .../storage/am/lsm/common/impls/LSMHarness.java    | 31 ++++++++++++++++------
 10 files changed, 71 insertions(+), 28 deletions(-)

diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
index e4ab8bdd16,abb41f2a0f..792fb4c4ea
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/IRecordConverter.java
@@@ -18,9 -18,24 +18,15 @@@
   */
  package org.apache.asterix.external.api;
  
+ import java.io.Closeable;
  import java.io.IOException;
 -import java.util.function.LongSupplier;
  
  @FunctionalInterface
- public interface IRecordConverter<I, O> {
+ public interface IRecordConverter<I, O> extends Closeable {
+ 
      O convert(IRawRecord<? extends I> input) throws IOException;
+ 
 -    /**
 -     * Configures the converter with information suppliers from the {@link IRecordReader} data source.
 -     *
 -     * @param lineNumber line number supplier
 -     */
 -    default void configure(LongSupplier lineNumber) {
 -    }
 -
+     default void close() {
+         // default no-op
+     }
  }
diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index b1660a500b,236d853b4c..09b2cecc9b
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@@ -19,12 -19,11 +19,15 @@@
  
  package org.apache.hyracks.storage.am.lsm.common.impls;
  
 +import static org.apache.hyracks.util.ExitUtil.EC_INCONSISTENT_STORAGE_REFERENCES;
 +
+ import java.time.ZoneId;
+ import java.time.ZonedDateTime;
+ import java.time.format.DateTimeFormatter;
  import java.util.ArrayList;
  import java.util.List;
 +import java.util.Set;
 +import java.util.concurrent.TimeUnit;
  import java.util.concurrent.atomic.AtomicBoolean;
  import java.util.function.Predicate;
  
@@@ -144,14 -136,11 +149,12 @@@ public class LSMHarness implements ILSM
          int numEntered = 0;
          boolean entranceSuccessful = false;
          try {
--            final int componentsCount = components.size();
--            for (int i = 0; i < componentsCount; i++) {
--                final ILSMComponent component = components.get(i);
++            for (final ILSMComponent component : components) {
                  boolean isMutableComponent = numEntered == 0 && component.getType() == LSMComponentType.MEMORY;
                  if (!component.threadEnter(opType, isMutableComponent)) {
 +                    if (dumpState) {
 +                        LOGGER.warn("couldn't enter component: {}", component.dumpState());
 +                    }
                      break;
                  }
                  numEntered++;
@@@ -262,13 -246,43 +265,13 @@@
                  }
              }
          } finally {
 -            /*
 -             * cleanup inactive disk components if any
 -             */
 -            if (inactiveDiskComponentsToBeDeleted != null) {
 -                try {
 -                    //schedule a replication job to delete these inactive disk components from replicas
 -                    if (replicationEnabled) {
 -                        lsmIndex.scheduleReplication(null, inactiveDiskComponentsToBeDeleted,
 -                                ReplicationOperation.DELETE, opType);
 -                    }
 -                    for (ILSMDiskComponent c : inactiveDiskComponentsToBeDeleted) {
 -                        c.deactivateAndDestroy();
 -                    }
 -                } catch (Throwable e) { // NOSONAR Log and re-throw
 -                    if (LOGGER.isWarnEnabled()) {
 -                        LOGGER.log(Level.WARN, "Failure scheduling replication or destroying merged component", e);
 -                    }
 -                    throw e; // NOSONAR: The last call in the finally clause
 -                }
 -            }
 +            // the memory components clean up must be done first to avoid any unexpected exceptions during the rest
 +            // of the finally block
              if (inactiveMemoryComponentsToBeCleanedUp != null) {
 -                for (ILSMMemoryComponent c : inactiveMemoryComponentsToBeCleanedUp) {
 -                    tracer.instant(c.toString(), traceCategory, Scope.p, lsmIndex::toString);
 -                    c.cleanup();
 -                    synchronized (opTracker) {
 -                        c.reset();
 -                        // Notify all waiting threads whenever the mutable component's state
 -                        // has changed to inactive. This is important because even though we switched
 -                        // the mutable components, it is possible that the component that we just
 -                        // switched to is still busy flushing its data to disk. Thus, the notification
 -                        // that was issued upon scheduling the flush is not enough.
 -                        opTracker.notifyAll(); // NOSONAR: Always called inside synchronized block
 -                    }
 -                }
 +                cleanupInactiveMemoryComponents(inactiveMemoryComponentsToBeCleanedUp);
              }
 -            if (opType == LSMOperationType.FLUSH) {
 -                ILSMMemoryComponent flushingComponent = (ILSMMemoryComponent) ctx.getComponentHolder().get(0);
 +            if (opType == LSMOperationType.FLUSH && !failedOperation) {
-                 ILSMMemoryComponent flushingComponent = (ILSMMemoryComponent) ctx.getComponentHolder().get(0);
++                ILSMMemoryComponent flushingComponent = (ILSMMemoryComponent) ctx.getComponentHolder().getFirst();
                  // We must call flushed without synchronizing on opTracker to avoid deadlocks
                  flushingComponent.flushed();
              }
@@@ -383,7 -387,7 +386,7 @@@
          }
          getAndEnterComponents(ctx, LSMOperationType.MODIFICATION, false);
          try {
--            AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0);
++            AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().getFirst();
              c.getMetadata().put(key, value);
              c.setModified();
          } finally {
@@@ -407,7 -411,7 +410,7 @@@
          }
          getAndEnterComponents(ctx, LSMOperationType.FORCE_MODIFICATION, false);
          try {
--            AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0);
++            AbstractLSMMemoryComponent c = (AbstractLSMMemoryComponent) ctx.getComponentHolder().getFirst();
              c.getMetadata().put(key, value);
              c.setModified();
          } finally {
@@@ -427,7 -431,7 +430,8 @@@
          try {
              lsmIndex.modify(ctx, tuple);
              // The mutable component is always in the first index.
--            AbstractLSMMemoryComponent mutableComponent = (AbstractLSMMemoryComponent) ctx.getComponentHolder().get(0);
++            AbstractLSMMemoryComponent mutableComponent =
++                    (AbstractLSMMemoryComponent) ctx.getComponentHolder().getFirst();
              mutableComponent.setModified();
          } catch (Exception e) {
              failedOperation = true;
@@@ -547,9 -549,11 +551,11 @@@
      }
  
      public void doIo(ILSMIOOperation operation) {
+         String origName = Thread.currentThread().getName();
          try {
+             Thread.currentThread().setName(threadName(operation));
              operation.getCallback().beforeOperation(operation);
 -            ILSMDiskComponent newComponent = operation.getIOOpertionType() == LSMIOOperationType.FLUSH
 +            ILSMDiskComponent newComponent = operation.getIOOperationType() == LSMIOOperationType.FLUSH
                      ? lsmIndex.flush(operation) : lsmIndex.merge(operation);
              operation.setNewComponent(newComponent);
              operation.getCallback().afterOperation(operation);
@@@ -569,10 -573,11 +575,11 @@@
                  operation.setStatus(LSMIOOperationStatus.FAILURE);
                  operation.setFailure(th);
                  if (LOGGER.isErrorEnabled()) {
 -                    LOGGER.log(Level.ERROR, "{} operation.afterFinalize failed on {}", operation.getIOOpertionType(),
 -                            lsmIndex, th);
 +                    LOGGER.error("{} operation.afterFinalize failed on {}", operation.getIOOperationType(), lsmIndex,
 +                            th);
                  }
              }
+             Thread.currentThread().setName(origName);
          }
          // if the operation failed, we need to cleanup files
          if (operation.getStatus() == LSMIOOperationStatus.FAILURE) {
@@@ -581,13 -585,19 +588,21 @@@
          }
      }
  
+     private static String threadName(ILSMIOOperation operation) {
 -        if (operation.getIOOpertionType() == LSMIOOperationType.NOOP) {
 -            return String.valueOf(operation.getIOOpertionType());
++        if (operation.getIOOperationType() == LSMIOOperationType.NOOP) {
++            return String.valueOf(operation.getIOOperationType());
+         }
 -        return operation.getIOOpertionType() + ":" + operation.getTarget().getRelativePath() + "@"
++        return operation.getIOOperationType() + ":" + operation.getTarget().getRelativePath() + "@"
+                 + OP_THREAD_TIMESTAMP.format(ZonedDateTime.now());
+     }
+ 
      @Override
      public void merge(ILSMIOOperation operation) throws HyracksDataException {
 -        LOGGER.debug("Started a merge operation for index {}", lsmIndex);
 +        List<ILSMComponent> mergingComponents = ((MergeOperation) operation).getMergingComponents();
 +        LOGGER.debug("Started MERGE operation {} of {} components for index: {}", operation, mergingComponents.size(),
 +                lsmIndex);
          synchronized (opTracker) {
 -            enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE);
 +            enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE, false);
          }
          ILSMDiskComponent newComponent;
          try {
@@@ -783,7 -792,7 +798,7 @@@
      @Override
      public void deleteComponents(ILSMIndexOperationContext ctx, Predicate<ILSMComponent> predicate)
              throws HyracksDataException {
--        ILSMIOOperation ioOperation = null;
++        ILSMIOOperation ioOperation;
          // We need to always start the component delete from current memory component.
          // This will ensure Primary and secondary component id still matches after component delete
          if (!lsmIndex.isMemoryComponentsAllocated()) {


(asterixdb) 18/25: [ASTERIXDB-3657][FAIL]: handle non-serializable exceptions

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 3050d373ed09956bad08eb88b75c4c35fe7a38f3
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Mon Oct 13 21:50:16 2025 +0300

    [ASTERIXDB-3657][FAIL]: handle non-serializable exceptions
    
    Ext-ref: MB-68744
    Change-Id: I55e69623068a6c1759803cc699417021910237fd
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20485
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Michael Blow <mb...@apache.org>
    Tested-by: Hussain Towaileb <hu...@gmail.com>
---
 .../asterix/hyracks/bootstrap/CCApplication.java   |  9 +++
 .../asterix/hyracks/bootstrap/NCApplication.java   |  9 +++
 .../external/input/HDFSDataSourceFactory.java      |  3 +-
 .../writer/HDFSExternalFileWriterFactory.java      |  2 +-
 ...ReplacementsAwareJavaSerializationProvider.java | 82 ++++++++++++++++++++++
 .../hyracks/api/util/JavaSerializationUtils.java   | 39 +++++++---
 .../apache/hyracks/util/ThrowingIOFunction.java    | 40 +++++++++++
 7 files changed, 174 insertions(+), 10 deletions(-)

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
index 4e35ade636..1af75c168c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/CCApplication.java
@@ -25,6 +25,7 @@ import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNEC
 import static org.apache.asterix.common.api.IClusterManagementWork.ClusterState.ACTIVE;
 import static org.apache.asterix.common.api.IClusterManagementWork.ClusterState.REBALANCE_REQUIRED;
 import static org.apache.asterix.common.api.IClusterManagementWork.ClusterState.SHUTTING_DOWN;
+import static org.apache.hyracks.api.util.JavaSerializationUtils.registerReplacement;
 import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
 
 import java.io.File;
@@ -120,6 +121,7 @@ import org.apache.hyracks.api.job.JobFlag;
 import org.apache.hyracks.api.job.resource.IJobCapacityController;
 import org.apache.hyracks.api.lifecycle.LifeCycleComponentManager;
 import org.apache.hyracks.api.result.IJobResultCallback;
+import org.apache.hyracks.api.util.JavaSerializationUtils.SerializableExceptionProxy;
 import org.apache.hyracks.control.cc.BaseCCApplication;
 import org.apache.hyracks.control.cc.ClusterControllerService;
 import org.apache.hyracks.control.common.controllers.CCConfig;
@@ -137,6 +139,8 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import com.azure.storage.blob.models.BlobStorageException;
+
 public class CCApplication extends BaseCCApplication {
 
     private static final Logger LOGGER = LogManager.getLogger();
@@ -154,6 +158,11 @@ public class CCApplication extends BaseCCApplication {
         ccServiceCtx.setThreadFactory(
                 new AsterixThreadFactory(ccServiceCtx.getThreadFactory(), new LifeCycleComponentManager()));
         validateEnvironment();
+        registerSerializationReplacements();
+    }
+
+    private void registerSerializationReplacements() {
+        registerReplacement(BlobStorageException.class, SerializableExceptionProxy::new);
     }
 
     @Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 0e6a04be20..da606c0d4c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -25,6 +25,7 @@ import static org.apache.asterix.common.utils.Servlets.QUERY_SERVICE;
 import static org.apache.asterix.common.utils.Servlets.QUERY_STATUS;
 import static org.apache.asterix.common.utils.Servlets.UDF;
 import static org.apache.asterix.common.utils.Servlets.UDF_RECOVERY;
+import static org.apache.hyracks.api.util.JavaSerializationUtils.registerReplacement;
 import static org.apache.hyracks.control.common.controllers.ControllerConfig.Option.CLOUD_DEPLOYMENT;
 
 import java.io.File;
@@ -102,6 +103,7 @@ import org.apache.hyracks.api.control.CcId;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.io.IFileDeviceResolver;
 import org.apache.hyracks.api.job.resource.NodeCapacity;
+import org.apache.hyracks.api.util.JavaSerializationUtils.SerializableExceptionProxy;
 import org.apache.hyracks.control.common.controllers.NCConfig;
 import org.apache.hyracks.control.nc.BaseNCApplication;
 import org.apache.hyracks.control.nc.NodeControllerService;
@@ -115,6 +117,8 @@ import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import com.azure.storage.blob.models.BlobStorageException;
+
 public class NCApplication extends BaseNCApplication {
     private static final Logger LOGGER = LogManager.getLogger();
     protected NCExtensionManager ncExtensionManager;
@@ -141,6 +145,11 @@ public class NCApplication extends BaseNCApplication {
                 new AsterixThreadFactory(ncServiceCtx.getThreadFactory(), ncServiceCtx.getLifeCycleComponentManager()));
         validateEnvironment();
         configurePersistedResourceRegistry();
+        registerSerializationReplacements();
+    }
+
+    private void registerSerializationReplacements() {
+        registerReplacement(BlobStorageException.class, SerializableExceptionProxy::new);
     }
 
     @Override
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
index 8af73428bb..b820147c93 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/HDFSDataSourceFactory.java
@@ -166,7 +166,8 @@ public class HDFSDataSourceFactory implements IRecordReaderFactory<Object>, IExt
         } catch (InterruptedException ex) {
             throw HyracksDataException.create(ex);
         } catch (IOException ex) {
-            throw CompilationException.create(ErrorCode.EXTERNAL_SOURCE_ERROR, ExceptionUtils.getMessageOrToString(ex));
+            throw CompilationException.create(ErrorCode.EXTERNAL_SOURCE_ERROR, ex,
+                    ExceptionUtils.getMessageOrToString(ex));
         }
     }
 
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/HDFSExternalFileWriterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/HDFSExternalFileWriterFactory.java
index 5720fc32a5..711cd0506d 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/HDFSExternalFileWriterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/writer/HDFSExternalFileWriterFactory.java
@@ -137,7 +137,7 @@ public class HDFSExternalFileWriterFactory implements IExternalFileWriterFactory
                 doValidate(testFs);
             }
         } catch (IOException ex) {
-            throw new CompilationException(ErrorCode.EXTERNAL_SINK_ERROR, ExceptionUtils.getMessageOrToString(ex));
+            throw new CompilationException(ErrorCode.EXTERNAL_SINK_ERROR, ex, ExceptionUtils.getMessageOrToString(ex));
         }
     }
 
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/ReplacementsAwareJavaSerializationProvider.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/ReplacementsAwareJavaSerializationProvider.java
new file mode 100644
index 0000000000..15557961a0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/ReplacementsAwareJavaSerializationProvider.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.comm;
+
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hyracks.util.ThrowingIOFunction;
+
+public class ReplacementsAwareJavaSerializationProvider implements IJavaSerializationProvider {
+    public static final ReplacementsAwareJavaSerializationProvider INSTANCE =
+            new ReplacementsAwareJavaSerializationProvider();
+    private static final Map<Class<?>, ThrowingIOFunction<Object, Object>> replacements = new ConcurrentHashMap<>();
+
+    private ReplacementsAwareJavaSerializationProvider() {
+    }
+
+    @Override
+    public ObjectOutputStream newObjectOutputStream(OutputStream out) throws IOException {
+        return new ReplacementsAwareObjectOutputStream(out);
+    }
+
+    public Map<Class<?>, ThrowingIOFunction<Object, Object>> getReplacements() {
+        return Collections.unmodifiableMap(replacements);
+    }
+
+    public void registerReplacement(Class<?> clazz, ThrowingIOFunction<Object, Object> replacementFunction) {
+        replacements.put(clazz, replacementFunction);
+    }
+
+    private static class ReplacementsAwareObjectOutputStream extends ObjectOutputStream {
+        public ReplacementsAwareObjectOutputStream(OutputStream out) throws IOException {
+            super(out);
+            enableReplaceObject(true);
+        }
+
+        @Override
+        protected Object replaceObject(Object object) throws IOException {
+            Class<?> clazz = object.getClass();
+            if (clazz.isSynthetic()) {
+                return super.replaceObject(object);
+            }
+
+            // try exact match first (fast path)
+            ThrowingIOFunction<Object, Object> replacementFunction = replacements.get(clazz);
+            if (replacementFunction != null) {
+                return replacementFunction.process(object);
+            }
+
+            // fallback: match by assignability (handles subclasses / interfaces)
+            for (Map.Entry<Class<?>, ThrowingIOFunction<Object, Object>> e : replacements.entrySet()) {
+                if (e.getKey().isInstance(object)) {
+                    INSTANCE.registerReplacement(clazz, e.getValue());
+                    return e.getValue().process(object);
+                }
+            }
+
+            INSTANCE.registerReplacement(clazz, super::replaceObject);
+            return super.replaceObject(object);
+        }
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/JavaSerializationUtils.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/JavaSerializationUtils.java
index 8e2420498e..f51feba61f 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/JavaSerializationUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/JavaSerializationUtils.java
@@ -28,12 +28,15 @@ import java.io.ObjectStreamClass;
 import java.io.Serializable;
 import java.lang.reflect.Modifier;
 import java.lang.reflect.Proxy;
+import java.util.Map;
 
-import org.apache.hyracks.api.comm.DefaultJavaSerializationProvider;
 import org.apache.hyracks.api.comm.IJavaSerializationProvider;
+import org.apache.hyracks.api.comm.ReplacementsAwareJavaSerializationProvider;
+import org.apache.hyracks.util.ThrowingIOFunction;
 
 public class JavaSerializationUtils {
-    private static IJavaSerializationProvider serProvider = DefaultJavaSerializationProvider.INSTANCE;
+    private static final ReplacementsAwareJavaSerializationProvider serProvider =
+            ReplacementsAwareJavaSerializationProvider.INSTANCE;
 
     private JavaSerializationUtils() {
     }
@@ -89,10 +92,6 @@ public class JavaSerializationUtils {
         return Class.forName(className);
     }
 
-    public static void setSerializationProvider(IJavaSerializationProvider serProvider) {
-        JavaSerializationUtils.serProvider = serProvider;
-    }
-
     public static IJavaSerializationProvider getSerializationProvider() {
         return serProvider;
     }
@@ -106,7 +105,7 @@ public class JavaSerializationUtils {
     }
 
     private static class ClassLoaderObjectInputStream extends ObjectInputStream {
-        private ClassLoader classLoader;
+        private final ClassLoader classLoader;
 
         protected ClassLoaderObjectInputStream(InputStream in, ClassLoader classLoader)
                 throws IOException, SecurityException {
@@ -120,7 +119,7 @@ public class JavaSerializationUtils {
         }
 
         @Override
-        protected Class<?> resolveProxyClass(String[] interfaces) throws IOException, ClassNotFoundException {
+        protected Class<?> resolveProxyClass(String[] interfaces) throws ClassNotFoundException {
             ClassLoader nonPublicLoader = null;
             boolean hasNonPublicInterface = false;
 
@@ -147,4 +146,28 @@ public class JavaSerializationUtils {
             }
         }
     }
+
+    public static Map<Class<?>, ThrowingIOFunction<Object, Object>> getReplacements() {
+        return serProvider.getReplacements();
+    }
+
+    public static <T> void registerReplacement(Class<T> clazz, ThrowingIOFunction<? super T, ?> replacementFunction) {
+        serProvider.registerReplacement(clazz, object -> replacementFunction.process(clazz.cast(object)));
+    }
+
+    public static class SerializableExceptionProxy extends Throwable {
+        private static final long serialVersionUID = 1L;
+        private final String type;
+
+        public SerializableExceptionProxy(Throwable t) {
+            super(ExceptionUtils.getMessageOrToString(t));
+            this.type = t.getClass().getName();
+            setStackTrace(t.getStackTrace());
+        }
+
+        @Override
+        public String toString() {
+            return type + ": " + getMessage();
+        }
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIOFunction.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIOFunction.java
new file mode 100644
index 0000000000..5829ae2cee
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThrowingIOFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+import java.io.IOException;
+import java.util.function.Function;
+
+import com.google.common.util.concurrent.UncheckedExecutionException;
+
+@FunctionalInterface
+public interface ThrowingIOFunction<I, R> {
+    R process(I input) throws IOException;
+
+    @SuppressWarnings("Duplicates")
+    static <I, R> Function<I, R> asUnchecked(ThrowingIOFunction<I, R> consumer) {
+        return input -> {
+            try {
+                return consumer.process(input);
+            } catch (Exception e) {
+                throw new UncheckedExecutionException(e);
+            }
+        };
+    }
+}


(asterixdb) 05/25: Merge branch 'gerrit/neo' into 'gerrit/trinity'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 909f6ce5c9c37d3a00328423b7d2f875d4253859
Merge: 99fa79cd50 4874143e7d
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sun Oct 5 11:36:17 2025 -0400

    Merge branch 'gerrit/neo' into 'gerrit/trinity'
    
     * [NO ISSUE][*DB][EXT] Make IDataParser extend Closeable, misc
    
    Ext-ref: MB-68827
    Change-Id: I434142a9b9cd2d1fc941b1e1f255ef740308a1ec

 .../java/org/apache/asterix/active/ActiveManager.java  |  1 +
 .../org/apache/asterix/external/api/IDataParser.java   | 13 +++++++++----
 .../apache/asterix/external/api/IRecordConverter.java  |  9 +++++++--
 .../dataflow/FeedRecordDataFlowController.java         |  2 ++
 .../data/std/util/ByteArrayAccessibleOutputStream.java | 18 ++++++++++++++++++
 .../org/apache/hyracks/http/server/utils/HttpUtil.java |  6 ++++--
 .../common/dataflow/IndexDropOperatorNodePushable.java |  2 +-
 .../storage/am/lsm/common/impls/LSMHarness.java        | 16 ++++++++++++++++
 8 files changed, 58 insertions(+), 9 deletions(-)

diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 8ad67f7f85,2ac29fe640..236d853b4c
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@@ -577,9 -584,19 +585,17 @@@ public class LSMHarness implements ILSM
          }
      }
  
+     private static String threadName(ILSMIOOperation operation) {
+         if (operation.getIOOpertionType() == LSMIOOperationType.NOOP) {
+             return String.valueOf(operation.getIOOpertionType());
+         }
+         return operation.getIOOpertionType() + ":" + operation.getTarget().getRelativePath() + "@"
+                 + OP_THREAD_TIMESTAMP.format(ZonedDateTime.now());
+     }
+ 
      @Override
      public void merge(ILSMIOOperation operation) throws HyracksDataException {
 -        if (LOGGER.isDebugEnabled()) {
 -            LOGGER.debug("Started a merge operation for index: {}", lsmIndex);
 -        }
 +        LOGGER.debug("Started a merge operation for index {}", lsmIndex);
          synchronized (opTracker) {
              enterComponents(operation.getAccessor().getOpContext(), LSMOperationType.MERGE);
          }


(asterixdb) 21/25: [ASTERIXDB-3649][*DB] Improve async request API

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 6589517084c00eb53392e01e35f9e1d32296edb1
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Thu Oct 2 12:46:20 2025 -0700

    [ASTERIXDB-3649][*DB] Improve async request API
    
    Change-Id: I25c0651531a8a48271c901a7368d8668874a9d18
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20378
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Reviewed-by: Ian Maxon <im...@apache.org>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../apache/asterix/translator/ClientRequest.java   |   4 +
 .../asterix/translator/IStatementExecutor.java     |  33 +++-
 .../apache/asterix/translator/ResultMetadata.java  |  38 +++-
 .../apache/asterix/translator/SessionConfig.java   |  21 ++-
 .../apache/asterix/api/common/APIFramework.java    |  13 +-
 .../api/http/server/AbstractQueryApiServlet.java   | 195 +++++++++++++++++++++
 .../api/http/server/NCQueryResultApiServlet.java   |  61 +++++++
 .../api/http/server/NCQueryServiceServlet.java     |   7 +-
 .../api/http/server/NCQueryStatusApiServlet.java   |  20 ++-
 .../api/http/server/QueryResultApiServlet.java     | 141 +++++++++------
 .../http/server/QueryServiceRequestParameters.java |  14 +-
 .../api/http/server/QueryServiceServlet.java       | 156 +++--------------
 .../api/http/server/QueryStatusApiServlet.java     | 115 ++++++++++--
 .../apache/asterix/api/http/server/ResultUtil.java |  21 +++
 .../app/message/ClientInfoRequestMessage.java      |  67 +++++++
 .../app/message/ClientInfoResponseMessage.java     |  50 ++++++
 .../DiscardResultPartitionRequestMessage.java      |  57 ++++++
 .../app/message/DiscardResultRequestMessage.java   |  62 +++++++
 .../app/message/DiscardResultResponseMessage.java  |  44 +++++
 .../asterix/app/result/JobResultCallback.java      |   3 +
 .../apache/asterix/app/result/ResultHandle.java    |  75 ++++++--
 .../apache/asterix/app/result/ResultReader.java    |  10 ++
 .../app/result/fields/CreatedAtPrinter.java        |  36 ++--
 .../asterix/app/result/fields/MetricsPrinter.java  | 151 +++++++++++-----
 .../asterix/app/result/fields/NcResultPrinter.java |  12 +-
 .../app/result/fields/PartitionInfoPrinter.java    |  64 +++++++
 .../app/result/fields/ResultCountPrinter.java      |  35 ++--
 .../asterix/app/translator/QueryTranslator.java    | 152 ++++++++++++----
 .../asterix/hyracks/bootstrap/NCApplication.java   |   8 +-
 .../apache/asterix/utils/AsyncRequestsAPIUtil.java | 151 ++++++++++++++++
 .../asterix/test/common/ResultExtractor.java       |   7 +-
 .../apache/asterix/test/common/TestExecutor.java   |  55 ++++--
 .../async-compilation-failed.1.async.sqlpp         |   1 -
 .../async-exhausted-result.1.async.sqlpp           |   1 -
 .../async-exhausted-result.2.pollget.http}         |   6 +-
 .../async-exhausted-result.3.get.http}             |   4 +-
 .../async-exhausted-result.4.delete.http}          |   4 +-
 .../async-exhausted-result.5.get.http}             |   4 +-
 .../async-failed/async-failed.1.async.sqlpp        |   1 -
 .../async-failed/async-failed.2.pollget.http}      |   5 +-
 .../async-json/async-json.1.async.sqlpp            |   1 -
 .../async-json/async-json.2.pollget.http}          |   5 +-
 .../async-json/async-json.3.get.http}              |   6 +-
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 -
 .../async-repeated/async-repeated.2.pollget.http}  |   5 +-
 .../async-repeated/async-repeated.3.get.http}      |   4 +-
 .../async-repeated/async-repeated.4.get.http}      |   4 +-
 .../async-repeated/async-repeated.5.query.sqlpp}   |   2 -
 .../async-running/async-running.1.async.sqlpp      |   1 -
 .../async-running/async-running.2.pollget.http}    |   4 +-
 .../async-running/async-running.3.pollget.http}    |   5 +-
 .../async-running/async-running.4.get.http}        |   4 +-
 .../async-timeout}/async.1.async.sqlpp             |   4 +-
 .../async-timeout/async.2.pollget.http}            |   4 +-
 .../async/async.1.async.sqlpp                      |   1 -
 .../async/async.2.pollget.http}                    |   5 +-
 .../async/async.3.get.http}                        |   4 +-
 .../deferred/deferred.1.deferred.sqlpp}            |   3 +-
 .../deferred/deferred.2.get.http}                  |   4 +-
 .../async-deferred/AsyncDeferredQueries.xml        |  52 ++++++
 .../async-compilation-failed.1.async.sqlpp         |   1 +
 .../async-exhausted-result.1.async.sqlpp           |   1 +
 .../async-failed/async-failed.1.async.sqlpp        |   1 +
 .../async-json/async-json.1.async.sqlpp            |   1 +
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 +
 .../async-running/async-running.1.async.sqlpp      |   1 +
 .../async-deferred/async/async.1.async.sqlpp       |   1 +
 .../queries_sqlpp/flwor/at00/at00.6.async.sqlpp    |   1 +
 .../async-compilation-failed.1.regex               |   2 +
 .../async-exhausted-result.1.ignore                |   0
 .../async-exhausted-result.2.regexjson             |   8 +
 .../async-exhausted-result.3.regexjson             |  17 ++
 .../async-exhausted-result.4.ignore                |  17 ++
 .../async-failed/async-failed.1.ignore             |   0
 .../async-failed/async-failed.2.regex              |   2 +
 .../async-json/async-json.1.ignore                 |   0
 .../async-json/async-json.2.regexjson              |   8 +
 .../async-json/async-json.3.json                   |   5 +
 .../async-repeated/async-repeated.1.ignore         |   0
 .../async-repeated/async-repeated.2.regexjson      |   8 +
 .../async-repeated/async-repeated.3.regexjson      |  17 ++
 .../async-repeated/async-repeated.4.regexjson      |  17 ++
 .../async-repeated/async-repeated.5.json           |  10 ++
 .../async-running/async-running.1.ignore           |   0
 .../async-running/async-running.2.json             |   3 +
 .../async-running/async-running.3.regexjson        |   8 +
 .../async-running/async-running.4.regexjson        |   8 +
 .../async-timeout/async-timeout.1.ignore           |   0
 .../async-timeout/async-timeout.2.regex            |   3 +
 .../async-deferred-improved/async/async.1.ignore   |   0
 .../async/async.2.regexjson                        |   8 +
 .../async/async.3.regexjson                        |  17 ++
 .../deferred/deferred.1.ignore                     |   0
 .../deferred/deferred.2.regexjson                  |  17 ++
 .../apache/asterix/common/api/IRequestTracker.java |   8 +
 .../asterix/runtime/utils/RequestTracker.java      |  18 ++
 .../api/result/IResultPartitionManager.java        |   3 +-
 .../org/apache/hyracks/api/result/IResultSet.java  |   3 +
 .../hyracks/api/result/IResultSetReader.java       |   2 +
 .../hyracks/api/result/ResultDirectoryRecord.java  |  23 ++-
 .../apache/hyracks/api/result/ResultJobRecord.java |  22 ++-
 .../client/result/PartitionResultSetReader.java    | 112 ++++++++++++
 .../apache/hyracks/client/result/ResultSet.java    |  13 ++
 .../hyracks/client/result/ResultSetReader.java     |  49 ++++--
 .../hyracks/control/cc/ClusterControllerIPCI.java  |   4 +-
 .../control/cc/result/IResultDirectoryService.java |   6 +-
 .../control/cc/result/ResultDirectoryService.java  |  24 ++-
 .../control/cc/work/GetResultMetadataWork.java     |   3 -
 .../work/RegisterResultPartitionLocationWork.java  |   7 +-
 .../ReportResultPartitionWriteCompletionWork.java  |   7 +-
 .../control/common/base/IClusterController.java    |   5 +-
 .../hyracks/control/common/ipc/CCNCFunctions.java  |  19 +-
 .../common/ipc/ClusterControllerRemoteProxy.java   |  10 +-
 .../control/nc/result/ResultPartitionManager.java  |   9 +-
 .../control/nc/result/ResultPartitionWriter.java   |   9 +-
 .../hyracks/control/nc/result/ResultState.java     |   9 +-
 .../hyracks/hyracks-dataflow-std/pom.xml           |   1 -
 .../std/result/ResultWriterOperatorDescriptor.java |   2 +
 118 files changed, 2095 insertions(+), 479 deletions(-)

diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ClientRequest.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ClientRequest.java
index bd6281402f..f7ddcf535e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ClientRequest.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ClientRequest.java
@@ -75,6 +75,10 @@ public class ClientRequest extends BaseClientRequest {
         setRunning();
     }
 
+    public JobId getJobId() {
+        return jobId;
+    }
+
     public Thread getExecutor() {
         return executor;
     }
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
index 55d3276dd1..41c153248f 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/IStatementExecutor.java
@@ -42,7 +42,6 @@ import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.om.base.IAObject;
 import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
-import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.client.IClusterInfoCollector;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
@@ -95,16 +94,42 @@ public interface IStatementExecutor {
     }
 
     class ResultMetadata implements Serializable {
-        private static final long serialVersionUID = 1L;
+        private static final long serialVersionUID = 2L;
 
-        private final List<Triple<JobId, ResultSetId, ARecordType>> resultSets = new ArrayList<>();
+        private final List<ResultSetInfo> resultSets = new ArrayList<>();
 
-        public List<Triple<JobId, ResultSetId, ARecordType>> getResultSets() {
+        public List<ResultSetInfo> getResultSets() {
             return resultSets;
         }
 
     }
 
+    class ResultSetInfo implements Serializable {
+        private static final long serialVersionUID = 1L;
+
+        private final JobId jobId;
+        private final ResultSetId resultSetId;
+        private final ARecordType recordType;
+
+        public ResultSetInfo(JobId jobId, ResultSetId resultSetId, ARecordType recordType) {
+            this.jobId = jobId;
+            this.resultSetId = resultSetId;
+            this.recordType = recordType;
+        }
+
+        public JobId getJobId() {
+            return jobId;
+        }
+
+        public ResultSetId getResultSetId() {
+            return resultSetId;
+        }
+
+        public ARecordType getRecordType() {
+            return recordType;
+        }
+    }
+
     class Stats implements Serializable {
         private static final long serialVersionUID = 5885273238208454612L;
 
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
index 5d7705ba4f..de6026b25a 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/ResultMetadata.java
@@ -27,7 +27,7 @@ import org.apache.hyracks.api.result.IResultMetadata;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
 public class ResultMetadata implements IResultMetadata {
-    private static final long serialVersionUID = 1905367559307369036L;
+    private static final long serialVersionUID = 1905367559307369037L;
 
     private final SessionConfig.OutputFormat format;
     private long jobDuration;
@@ -43,6 +43,10 @@ public class ResultMetadata implements IResultMetadata {
     private long cloudReadRequestsCount;
     private long cloudPagesReadCount;
     private long cloudPagesPersistedCount;
+    private long resultCount;
+    private long compileTime;
+    private long createTime;
+    private long endTime;
 
     public ResultMetadata(SessionConfig.OutputFormat format) {
         this.format = format;
@@ -158,6 +162,38 @@ public class ResultMetadata implements IResultMetadata {
         this.queueWaitTimeInNanos = queueWaitTimeInNanos;
     }
 
+    public void setResultCount(long resultCount) {
+        this.resultCount = resultCount;
+    }
+
+    public long getResultCount() {
+        return resultCount;
+    }
+
+    public long getCompileTime() {
+        return compileTime;
+    }
+
+    public void setCompileTime(long compileTime) {
+        this.compileTime = compileTime;
+    }
+
+    public long getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(long createTime) {
+        this.createTime = createTime;
+    }
+
+    public long getEndTime() {
+        return endTime;
+    }
+
+    public void setEndTime(long endTime) {
+        this.endTime = endTime;
+    }
+
     @Override
     public String toString() {
         return "ResultMetadata{" + "format=" + format + ", jobDuration=" + jobDuration + ", processedObjects="
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
index b7eb98b13f..fcd4325368 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SessionConfig.java
@@ -40,7 +40,7 @@ import org.apache.logging.log4j.Logger;
  * <li>It allows you to specify output format-specific parameters.
  */
 public class SessionConfig implements Serializable {
-    private static final long serialVersionUID = 2L;
+    private static final long serialVersionUID = 3L;
 
     /**
      * Used to specify the output format for the primary execution.
@@ -166,6 +166,8 @@ public class SessionConfig implements Serializable {
     private final boolean generateJobSpec;
     private final boolean optimize;
     private long maxWarnings;
+    private long timeout;
+    private boolean includeHost;
 
     // Flags.
     private final Map<String, Boolean> flags;
@@ -210,6 +212,7 @@ public class SessionConfig implements Serializable {
         this.planFormat = planFormat;
         this.clientType = ClientType.ASTERIX;
         this.hyracksJobFormat = jobFormat;
+        this.includeHost = false;
     }
 
     /**
@@ -267,6 +270,22 @@ public class SessionConfig implements Serializable {
         this.maxWarnings = maxWarnings;
     }
 
+    public void setTimeout(long timeout) {
+        this.timeout = timeout;
+    }
+
+    public long getTimeout() {
+        return timeout;
+    }
+
+    public void setIncludeHost(boolean includeHost) {
+        this.includeHost = includeHost;
+    }
+
+    public boolean isIncludeHost() {
+        return includeHost;
+    }
+
     /**
      * Retrieve the value of the "execute query" flag.
      */
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 511861849c..3e0677e2b9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -219,6 +219,15 @@ public class APIFramework {
             Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer, IWarningCollector warningCollector,
             IRequestParameters requestParameters, EnumSet<JobFlag> runtimeFlags)
             throws AlgebricksException, ACIDException {
+        return compileQuery(clusterInfoCollector, metadataProvider, query, varCounter, outputDatasetName, output,
+                statement, externalVars, printer, warningCollector, requestParameters, runtimeFlags, null);
+    }
+
+    public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
+            Query query, int varCounter, String outputDatasetName, SessionOutput output, ICompiledStatement statement,
+            Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer, IWarningCollector warningCollector,
+            IRequestParameters requestParameters, EnumSet<JobFlag> runtimeFlags, ResultMetadata resultMetadata)
+            throws AlgebricksException, ACIDException {
 
         try {
             // establish facts
@@ -239,7 +248,9 @@ public class APIFramework {
             metadataProvider.setTxnId(txnId);
             ILangExpressionToPlanTranslator t =
                     translatorFactory.createExpressionToPlanTranslator(metadataProvider, varCounter, externalVars);
-            ResultMetadata resultMetadata = new ResultMetadata(output.config().fmt());
+            if (resultMetadata == null) {
+                resultMetadata = new ResultMetadata(output.config().fmt());
+            }
             ILogicalPlan plan = isLoad || isCopy ? t.translateCopyOrLoad((ICompiledDmlStatement) statement)
                     : t.translate(query, outputDatasetName, statement, resultMetadata);
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
index bf77c24f96..06c83cd8f9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/AbstractQueryApiServlet.java
@@ -20,22 +20,42 @@ package org.apache.asterix.api.http.server;
 
 import static org.apache.asterix.api.http.server.ServletConstants.HYRACKS_CONNECTION_ATTR;
 
+import java.util.Collections;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentMap;
 
+import org.apache.asterix.app.result.ExecutionError;
+import org.apache.asterix.app.result.ResponsePrinter;
+import org.apache.asterix.app.result.ResultHandle;
+import org.apache.asterix.app.result.fields.ErrorsPrinter;
 import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.api.IClientRequest;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.translator.ClientRequest;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.client.IHyracksClientConnection;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IError;
+import org.apache.hyracks.api.exceptions.IFormattedException;
+import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.result.IResultSet;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.AbstractServlet;
+import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import io.netty.handler.codec.http.HttpResponseStatus;
+
 public class AbstractQueryApiServlet extends AbstractServlet {
     private static final Logger LOGGER = LogManager.getLogger();
     protected final IApplicationContext appCtx;
 
     public enum ResultStatus {
+        QUEUED("queued"),
         RUNNING("running"),
         SUCCESS("success"),
         TIMEOUT("timeout"),
@@ -69,4 +89,179 @@ public class AbstractQueryApiServlet extends AbstractServlet {
         }
         return hcc;
     }
+
+    protected void handleExecuteStatementException(Throwable t,
+            QueryServiceServlet.RequestExecutionState executionState, QueryServiceRequestParameters param,
+            IServletResponse response) {
+        handleExecuteStatementExceptionInternal(t, executionState, param.getRequestId(), param.getClientContextID(),
+                param, response);
+    }
+
+    protected void handleExecuteStatementException(Throwable t,
+            QueryServiceServlet.RequestExecutionState executionState, String requestId, String clientContextId,
+            IServletResponse response) {
+        handleExecuteStatementExceptionInternal(t, executionState, requestId, clientContextId, null, response);
+    }
+
+    private void handleExecuteStatementExceptionInternal(Throwable t,
+            QueryServiceServlet.RequestExecutionState executionState, String requestId, String clientContextId,
+            QueryServiceRequestParameters param, IServletResponse response) {
+        if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) {
+            if (LOGGER.isDebugEnabled()) {
+                logException(Level.DEBUG, t.getMessage(), requestId, clientContextId, t);
+            } else {
+                logException(Level.INFO, t.getMessage(), requestId, clientContextId);
+            }
+            executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
+            return;
+        } else if (t instanceof IFormattedException) {
+            IFormattedException formattedEx = (IFormattedException) t;
+            Optional<IError> maybeError = formattedEx.getError();
+            if (maybeError.isPresent()) {
+                IError error = maybeError.get();
+                boolean handled =
+                        (param != null) ? handleIFormattedException(error, formattedEx, executionState, param, response)
+                                : handleIFormattedException(error, formattedEx, executionState, requestId,
+                                        clientContextId, response);
+                if (handled) {
+                    return;
+                }
+            }
+        }
+        logException(Level.WARN, "unexpected exception", requestId, clientContextId, t);
+        executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
+    }
+
+    protected void logException(Level lvl, String msg, String clientCtxId, String uuid) {
+        LOGGER.log(lvl, "handleException: {}: uuid={}, clientContextID={}", msg, uuid, clientCtxId);
+    }
+
+    protected void logException(Level lvl, String msg, String clientCtxId, String uuid, Throwable t) {
+        LOGGER.log(lvl, "handleException: {}: uuid={}, clientContextID={}", msg, uuid, clientCtxId, t);
+    }
+
+    protected boolean handleIFormattedException(IError error, IFormattedException ex,
+            QueryServiceServlet.RequestExecutionState executionState, QueryServiceRequestParameters param,
+            IServletResponse response) {
+        return handleIFormattedException(error, ex, executionState, param.getRequestId(), param.getClientContextID(),
+                response);
+    }
+
+    protected boolean handleIFormattedException(IError error, IFormattedException ex,
+            QueryServiceServlet.RequestExecutionState executionState, String requestId, String clientContextId,
+            IServletResponse response) {
+        if (ErrorCode.ASTERIX.equals(error.component())) {
+            switch ((ErrorCode) error) {
+                case INVALID_REQ_PARAM_VAL:
+                case INVALID_REQ_JSON_VAL:
+                case NO_STATEMENT_PROVIDED:
+                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
+                    return true;
+                case REQUEST_TIMEOUT:
+                    logException(Level.INFO, "request execution timed out", requestId, clientContextId);
+                    executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
+                    return true;
+                case REJECT_NODE_UNREGISTERED:
+                case REJECT_BAD_CLUSTER_STATE:
+                    logException(Level.WARN, ex.getMessage(), requestId, clientContextId);
+                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
+                    return true;
+                default:
+                    // fall-through
+            }
+        } else if (error instanceof org.apache.hyracks.api.exceptions.ErrorCode) {
+            switch ((org.apache.hyracks.api.exceptions.ErrorCode) error) {
+                case JOB_REQUIREMENTS_EXCEED_CAPACITY:
+                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
+                    return true;
+            }
+        }
+        return false;
+    }
+
+    protected void requestFailed(Throwable throwable, ResponsePrinter responsePrinter,
+            QueryServiceServlet.RequestExecutionState executionState) {
+        final ExecutionError executionError = ExecutionError.of(throwable);
+        responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(executionError)));
+    }
+
+    protected ResultHandle parseAndValidateHandle(IServletRequest request, IServletResponse response) {
+        final String strHandle = localPath(request);
+        final ResultHandle handle = ResultHandle.parse(strHandle);
+        if (handle == null) {
+            response.setStatus(HttpResponseStatus.BAD_REQUEST);
+            return null;
+        }
+        try {
+            if (handle.getRequestId() != null
+                    && !isValidRequest(handle.getRequestId(), handle.getJobId(), request, response)) {
+                return null;
+            }
+        } catch (HyracksDataException e) {
+            response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+            return null;
+        }
+        return handle;
+    }
+
+    protected boolean isValidRequest(String requestId, JobId jobId, IServletRequest request, IServletResponse response)
+            throws HyracksDataException {
+        Optional<IClientRequest> clientRequest =
+                ((ICcApplicationContext) appCtx).getRequestTracker().getAsyncOrDeferredRequest(requestId);
+        if (clientRequest.isEmpty() || ((ClientRequest) clientRequest.get()).getJobId() == null) {
+            response.setStatus(HttpResponseStatus.NOT_FOUND);
+            return false;
+        }
+        return true;
+    }
+
+    protected static class RequestExecutionState {
+        private long execStart = -1;
+        private long execEnd = -1;
+        private ResultStatus resultStatus = ResultStatus.FATAL;
+        private HttpResponseStatus httpResponseStatus = HttpResponseStatus.INTERNAL_SERVER_ERROR;
+
+        public void setStatus(ResultStatus resultStatus, HttpResponseStatus httpResponseStatus) {
+            this.resultStatus = resultStatus;
+            this.httpResponseStatus = httpResponseStatus;
+        }
+
+        public ResultStatus getResultStatus() {
+            return resultStatus;
+        }
+
+        public HttpResponseStatus getHttpStatus() {
+            return httpResponseStatus;
+        }
+
+        void start() {
+            execStart = System.nanoTime();
+        }
+
+        void end() {
+            execEnd = System.nanoTime();
+        }
+
+        void finish() {
+            if (execStart == -1) {
+                execEnd = -1;
+            } else if (execEnd == -1) {
+                execEnd = System.nanoTime();
+            }
+        }
+
+        public long duration() {
+            return execEnd - execStart;
+        }
+
+        protected StringBuilder append(StringBuilder sb) {
+            return sb.append("ResultStatus: ").append(resultStatus.str()).append(" HTTPStatus: ")
+                    .append(String.valueOf(httpResponseStatus));
+        }
+
+        @Override
+        public String toString() {
+            return append(new StringBuilder()).toString();
+        }
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryResultApiServlet.java
new file mode 100644
index 0000000000..c9972fc9c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryResultApiServlet.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.api.http.server;
+
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.asterix.app.message.DiscardResultRequestMessage;
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.asterix.utils.AsyncRequestsAPIUtil;
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
+
+public class NCQueryResultApiServlet extends QueryResultApiServlet {
+
+    public NCQueryResultApiServlet(ConcurrentMap<String, Object> ctx, IApplicationContext appCtx, String... paths) {
+        super(ctx, appCtx, paths);
+    }
+
+    @Override
+    protected boolean isValidRequest(String requestId, JobId jobId, IServletRequest request, IServletResponse response)
+            throws HyracksDataException {
+        return AsyncRequestsAPIUtil.isValidRequest(appCtx, requestId, jobId, response);
+    }
+
+    @Override
+    protected void discardResult(String requestId, JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+        INCServiceContext serviceCtx = (INCServiceContext) appCtx.getServiceContext();
+        INCMessageBroker messageBroker = (INCMessageBroker) serviceCtx.getMessageBroker();
+        MessageFuture messageFuture = messageBroker.registerMessageFuture();
+        long futureId = messageFuture.getFutureId();
+        DiscardResultRequestMessage request =
+                new DiscardResultRequestMessage(serviceCtx.getNodeId(), futureId, jobId, resultSetId, requestId);
+        try {
+            messageBroker.sendMessageToPrimaryCC(request);
+        } catch (Exception e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
index e0e3cbc4b5..5ad2d1caa0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryServiceServlet.java
@@ -84,7 +84,12 @@ public class NCQueryServiceServlet extends QueryServiceServlet {
         final IStatementExecutor.ResultDelivery delivery = resultProperties.getDelivery();
         ExecuteStatementResponseMessage responseMsg;
         MessageFuture responseFuture = ncMb.registerMessageFuture();
-        final String handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
+        final String handleUrl;
+        if (!param.isIncludeHost()) {
+            handleUrl = getHandleUrl(param.getPath(), delivery);
+        } else {
+            handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
+        }
         try {
             long timeout = param.getTimeout();
             int stmtCategoryRestrictionMask = org.apache.asterix.app.translator.RequestParameters
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryStatusApiServlet.java
similarity index 53%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryStatusApiServlet.java
index ff2e48a349..4c0ce1df5a 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/NCQueryStatusApiServlet.java
@@ -16,11 +16,25 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.api.result;
+package org.apache.asterix.api.http.server;
 
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.utils.AsyncRequestsAPIUtil;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.http.api.IServletRequest;
+import org.apache.hyracks.http.api.IServletResponse;
+
+public class NCQueryStatusApiServlet extends QueryStatusApiServlet {
+
+    public NCQueryStatusApiServlet(ConcurrentMap<String, Object> ctx, IApplicationContext appCtx, String... paths) {
+        super(ctx, appCtx, paths);
+    }
 
-public interface IResultSet {
-    IResultSetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException;
+    protected boolean isValidRequest(String requestId, JobId jobId, IServletRequest request, IServletResponse response)
+            throws HyracksDataException {
+        return AsyncRequestsAPIUtil.isValidRequest(appCtx, requestId, jobId, response);
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
index 09ea35eae4..21f2267a6c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryResultApiServlet.java
@@ -25,18 +25,23 @@ import org.apache.asterix.app.result.ResponseMetrics;
 import org.apache.asterix.app.result.ResponsePrinter;
 import org.apache.asterix.app.result.ResultHandle;
 import org.apache.asterix.app.result.ResultReader;
+import org.apache.asterix.app.result.fields.CreatedAtPrinter;
 import org.apache.asterix.app.result.fields.MetricsPrinter;
 import org.apache.asterix.app.result.fields.ProfilePrinter;
 import org.apache.asterix.app.result.fields.ResultsPrinter;
 import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.translator.IStatementExecutor.Stats;
 import org.apache.asterix.translator.ResultMetadata;
 import org.apache.asterix.translator.SessionConfig;
 import org.apache.asterix.translator.SessionOutput;
+import org.apache.asterix.utils.AsyncRequestsAPIUtil;
 import org.apache.hyracks.api.exceptions.ErrorCode;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.result.IResultSet;
 import org.apache.hyracks.api.result.ResultJobRecord;
+import org.apache.hyracks.api.result.ResultSetId;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
@@ -54,91 +59,126 @@ public class QueryResultApiServlet extends AbstractQueryApiServlet {
     }
 
     @Override
-    protected void get(IServletRequest request, IServletResponse response) throws Exception {
-        long elapsedStart = System.nanoTime();
+    protected void delete(IServletRequest request, IServletResponse response) throws Exception {
         HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
         final String strHandle = localPath(request);
-        final ResultHandle handle = ResultHandle.parse(strHandle);
+        final ResultHandle handle = parseAndValidateHandle(request, response);
         if (handle == null) {
+            return; // Response status already set in parseAndValidateHandle
+        } else if (handle.getPartition() != -1) {
+            // Can only discard entire result sets, not individual partitions
             response.setStatus(HttpResponseStatus.BAD_REQUEST);
             return;
         }
+        try {
+            discardResult(handle.getRequestId(), handle.getJobId(), handle.getResultSetId());
+            response.setStatus(HttpResponseStatus.ACCEPTED);
+        } catch (HyracksDataException e) {
+            if (e.matches(ErrorCode.NO_RESULT_SET)) {
+                LOGGER.log(Level.INFO, "No results found for handle: \"{}\"", strHandle);
+                response.setStatus(HttpResponseStatus.NOT_FOUND);
+                return;
+            }
+            LOGGER.log(Level.WARN, "unexpected exception thrown from discard result", e);
+            response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+        } catch (Exception e) {
+            LOGGER.log(Level.WARN, "unexpected exception thrown from discard result", e);
+            response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
+        }
+    }
+
+    protected void discardResult(String requestId, JobId jobId, ResultSetId resultSetId) throws HyracksDataException {
+        AsyncRequestsAPIUtil.discardResultPartitions((ICcApplicationContext) appCtx, jobId, resultSetId, requestId);
+    }
+
+    @Override
+    protected void get(IServletRequest request, IServletResponse response) throws Exception {
+        HttpUtil.setContentType(response, HttpUtil.ContentType.TEXT_HTML, request);
+        final String strHandle = localPath(request);
+        final ResultHandle handle = parseAndValidateHandle(request, response);
+        if (handle == null) {
+            return; // Response status already set in parseAndValidateHandle
+        }
         IResultSet resultSet = getResultSet();
-        ResultReader resultReader = new ResultReader(resultSet, handle.getJobId(), handle.getResultSetId());
-        boolean printStarted = false;
-        ResponsePrinter printer = null;
+        ResultReader resultReader = handle.getPartition() != -1
+                ? new ResultReader(resultSet, handle.getJobId(), handle.getResultSetId(), handle.getPartition())
+                : new ResultReader(resultSet, handle.getJobId(), handle.getResultSetId());
         try {
             ResultJobRecord.Status status = resultReader.getStatus();
-            final HttpResponseStatus httpStatus;
-            if (status == null) {
-                httpStatus = HttpResponseStatus.NOT_FOUND;
-            } else {
-                switch (status.getState()) {
-                    case SUCCESS:
-                        httpStatus = HttpResponseStatus.OK;
-                        break;
-                    case RUNNING:
-                    case IDLE:
-                    case FAILED:
-                    case REMOVED:
-                        httpStatus = HttpResponseStatus.NOT_FOUND;
-                        break;
-                    default:
-                        httpStatus = HttpResponseStatus.INTERNAL_SERVER_ERROR;
-                        break;
-                }
-            }
+            final HttpResponseStatus httpStatus = ResultUtil.getHttpStatusFromResultStatus(status);
             response.setStatus(httpStatus);
             if (httpStatus != HttpResponseStatus.OK) {
                 return;
             }
             ResultMetadata metadata = (ResultMetadata) resultReader.getMetadata();
             SessionOutput sessionOutput = initResponse(request, response, metadata.getFormat());
-            printer = new ResponsePrinter(sessionOutput);
+            processResults(handle, resultReader, sessionOutput, metadata, request);
+        } catch (HyracksDataException e) {
+            if (e.matches(ErrorCode.NO_RESULT_SET)) {
+                LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
+                response.setStatus(HttpResponseStatus.NOT_FOUND);
+                return;
+            }
+            response.setStatus(HttpResponseStatus.BAD_REQUEST);
+            LOGGER.log(Level.WARN, "Error retrieving result for handle: \"{}\"", handle, e);
+        } catch (Exception e) {
+            response.setStatus(HttpResponseStatus.BAD_REQUEST);
+            LOGGER.log(Level.WARN, "Error retrieving result for handle: \"{}\"", handle, e);
+        }
+        if (response.writer().checkError()) {
+            LOGGER.warn("Error flushing output writer for \"{}\"", strHandle);
+        }
+    }
+
+    private void processResults(ResultHandle handle, ResultReader resultReader, SessionOutput sessionOutput,
+            ResultMetadata metadata, IServletRequest request) throws HyracksDataException {
+        ResponsePrinter printer = new ResponsePrinter(sessionOutput);
+        final Stats stats = new Stats();
+        boolean printStarted = false;
+        try {
             if (metadata.getFormat() == SessionConfig.OutputFormat.CLEAN_JSON
                     || metadata.getFormat() == SessionConfig.OutputFormat.LOSSLESS_JSON
                     || metadata.getFormat() == SessionConfig.OutputFormat.LOSSLESS_ADM_JSON) {
-                final Stats stats = new Stats();
                 printer.begin();
                 printStarted = true;
                 printer.addResultPrinter(new ResultsPrinter(appCtx, resultReader, null, stats, sessionOutput));
                 printer.printResults();
-                ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime() - elapsedStart,
-                        metadata.getJobDuration(), stats.getCount(), stats.getSize(), metadata.getProcessedObjects(), 0,
-                        metadata.getTotalWarningsCount(), stats.getCompileTime(), stats.getQueueWaitTime(),
-                        stats.getBufferCacheHitRatio(), stats.getBufferCachePageReadCount(),
-                        stats.getCloudReadRequestsCount(), stats.getCloudPagesReadCount(),
-                        stats.getCloudPagesPersistedCount());
+                ResponseMetrics metrics = buildMetrics(stats, metadata);
                 printer.addFooterPrinter(new MetricsPrinter(metrics, HttpUtil.getPreferredCharset(request)));
                 if (metadata.getJobProfile() != null) {
                     printer.addFooterPrinter(new ProfilePrinter(metadata.getJobProfile()));
                 }
+                if (handle.getRequestId() != null) {
+                    printer.addFooterPrinter(new CreatedAtPrinter(metadata.getCreateTime()));
+                }
                 printer.printFooters();
                 printer.end();
                 printStarted = false;
             } else {
-                ResultUtil.printResults(appCtx, resultReader, sessionOutput, new Stats(), null);
+                ResultUtil.printResults(appCtx, resultReader, sessionOutput, stats, null);
             }
-        } catch (HyracksDataException e) {
-            if (e.matches(ErrorCode.NO_RESULT_SET)) {
-                LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
-                response.setStatus(HttpResponseStatus.NOT_FOUND);
-                return;
-            }
-            response.setStatus(HttpResponseStatus.BAD_REQUEST);
-            response.writer().println(e.getMessage());
-            LOGGER.log(Level.WARN, "Error retrieving result for \"" + strHandle + "\"", e);
-        } catch (Exception e) {
-            response.setStatus(HttpResponseStatus.BAD_REQUEST);
-            LOGGER.log(Level.WARN, "Error retrieving result for \"" + strHandle + "\"", e);
         } finally {
-            if (printStarted && printer != null) {
+            if (printStarted) {
                 printer.end();
             }
         }
-        if (response.writer().checkError()) {
-            LOGGER.warn("Error flushing output writer for \"" + strHandle + "\"");
-        }
+    }
+
+    private ResponseMetrics buildMetrics(Stats stats, ResultMetadata metadata) {
+        long endTime = System.currentTimeMillis();
+        stats.setProcessedObjects(metadata.getProcessedObjects());
+        stats.setQueueWaitTime(metadata.getQueueWaitTimeInNanos());
+        stats.setBufferCacheHitRatio(metadata.getBufferCacheHitRatio());
+        stats.setBufferCachePageReadCount(metadata.getBufferCachePageReadCount());
+        stats.setCloudReadRequestsCount(metadata.getCloudReadRequestsCount());
+        stats.setCloudPagesReadCount(metadata.getCloudPagesReadCount());
+        stats.setCloudPagesPersistedCount(metadata.getCloudPagesPersistedCount());
+        stats.updateTotalWarningsCount(metadata.getTotalWarningsCount());
+        return ResponseMetrics.of(endTime - metadata.getCreateTime(), metadata.getJobDuration(), stats.getCount(),
+                stats.getSize(), metadata.getProcessedObjects(), 0, metadata.getTotalWarningsCount(),
+                metadata.getCompileTime(), stats.getQueueWaitTime(), stats.getBufferCacheHitRatio(),
+                stats.getBufferCachePageReadCount(), stats.getCloudReadRequestsCount(), stats.getCloudPagesReadCount(),
+                stats.getCloudPagesPersistedCount());
     }
 
     /**
@@ -206,5 +246,4 @@ public class QueryResultApiServlet extends AbstractQueryApiServlet {
         }
         return new SessionOutput(sessionConfig, response.writer(), resultPrefix, resultPostfix, appendHandle, null);
     }
-
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index de5ac26cf1..49aba9cbfe 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -84,7 +84,8 @@ public class QueryServiceRequestParameters {
         MULTI_STATEMENT("multi-statement"),
         MAX_WARNINGS("max-warnings"),
         SQL_COMPAT("sql-compat"),
-        SOURCE("source");
+        SOURCE("source"),
+        INCLUDE_HOST("include-host");
 
         private final String str;
 
@@ -156,6 +157,7 @@ public class QueryServiceRequestParameters {
     private long timeout = TimeUnit.MILLISECONDS.toMillis(Long.MAX_VALUE);
     private long maxResultReads = 1L;
     private long maxWarnings = 0L;
+    private boolean includeHost = false;
 
     public String getHost() {
         return host;
@@ -210,6 +212,14 @@ public class QueryServiceRequestParameters {
         this.timeout = timeout;
     }
 
+    public boolean isIncludeHost() {
+        return includeHost;
+    }
+
+    public void setIncludeHost(boolean includeHost) {
+        this.includeHost = includeHost;
+    }
+
     public boolean isPretty() {
         return pretty;
     }
@@ -531,6 +541,8 @@ public class QueryServiceRequestParameters {
         setClientType(parseIfExists(req, Parameter.CLIENT_TYPE.str(), valGetter, getClientType(), clientTypes::get));
         setSQLCompatMode(parseBoolean(req, Parameter.SQL_COMPAT.str(), valGetter, isSQLCompatMode()));
         setSource(valGetter.apply(req, Parameter.SOURCE.str()));
+        setIncludeHost(parseBoolean(req, Parameter.INCLUDE_HOST.str(), valGetter,
+                isIncludeHost() || getMode() == ResultDelivery.DEFERRED));
     }
 
     protected void setExtraParams(JsonNode jsonRequest) throws HyracksDataException {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 2b78498123..c0de06abee 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -26,22 +26,18 @@ import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.charset.Charset;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
-import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
 import java.util.function.Function;
 
 import org.apache.asterix.algebra.base.ILangExtension;
-import org.apache.asterix.app.result.ExecutionError;
 import org.apache.asterix.app.result.ExecutionWarning;
 import org.apache.asterix.app.result.ResponseMetrics;
 import org.apache.asterix.app.result.ResponsePrinter;
 import org.apache.asterix.app.result.fields.ClientContextIdPrinter;
-import org.apache.asterix.app.result.fields.ErrorsPrinter;
 import org.apache.asterix.app.result.fields.MetricsPrinter;
 import org.apache.asterix.app.result.fields.ParseOnlyResultPrinter;
 import org.apache.asterix.app.result.fields.PlansPrinter;
@@ -60,7 +56,6 @@ import org.apache.asterix.common.api.IRequestReference;
 import org.apache.asterix.common.context.IStorageComponentProvider;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.CompilationException;
-import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.compiler.provider.ILangCompilationProvider;
 import org.apache.asterix.hyracks.bootstrap.ApplicationConfigurator;
@@ -83,15 +78,12 @@ import org.apache.asterix.translator.SessionOutput;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.application.IServiceContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IError;
-import org.apache.hyracks.api.exceptions.IFormattedException;
 import org.apache.hyracks.api.exceptions.Warning;
 import org.apache.hyracks.api.result.IResultSet;
 import org.apache.hyracks.control.common.controllers.CCConfig;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -153,56 +145,6 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
         response.setStatus(HttpResponseStatus.OK);
     }
 
-    protected static class RequestExecutionState {
-        private long execStart = -1;
-        private long execEnd = -1;
-        private ResultStatus resultStatus = ResultStatus.FATAL;
-        private HttpResponseStatus httpResponseStatus = HttpResponseStatus.INTERNAL_SERVER_ERROR;
-
-        public void setStatus(ResultStatus resultStatus, HttpResponseStatus httpResponseStatus) {
-            this.resultStatus = resultStatus;
-            this.httpResponseStatus = httpResponseStatus;
-        }
-
-        public ResultStatus getResultStatus() {
-            return resultStatus;
-        }
-
-        public HttpResponseStatus getHttpStatus() {
-            return httpResponseStatus;
-        }
-
-        void start() {
-            execStart = System.nanoTime();
-        }
-
-        void end() {
-            execEnd = System.nanoTime();
-        }
-
-        void finish() {
-            if (execStart == -1) {
-                execEnd = -1;
-            } else if (execEnd == -1) {
-                execEnd = System.nanoTime();
-            }
-        }
-
-        public long duration() {
-            return execEnd - execStart;
-        }
-
-        protected StringBuilder append(StringBuilder sb) {
-            return sb.append("ResultStatus: ").append(resultStatus.str()).append(" HTTPStatus: ")
-                    .append(String.valueOf(httpResponseStatus));
-        }
-
-        @Override
-        public String toString() {
-            return append(new StringBuilder()).toString();
-        }
-    }
-
     private static SessionOutput createSessionOutput(PrintWriter resultWriter) {
         SessionOutput.ResultDecorator resultPrefix = ResultUtil.createPreResultDecorator();
         SessionOutput.ResultDecorator resultPostfix = ResultUtil.createPostResultDecorator();
@@ -242,18 +184,20 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
     }
 
     /**
-     * Determines the URL for a result handle based on the host and the path of the incoming request and the result
+     * Determines the URI for a result handle based on the result
      * delivery mode. Usually there will be a "status" endpoint for ASYNC requests that exposes the status of the
      * execution and a "result" endpoint for DEFERRED requests that will deliver the result for a successful execution.
      *
-     * @param host
-     *            hostname used for this request
      * @param path
      *            servlet path for this request
      * @param delivery
      *            ResultDelivery mode for this request
      * @return a handle (URL) that allows a client to access further information for this request
      */
+    protected String getHandleUrl(String path, ResultDelivery delivery) {
+        return path + handlePath(delivery);
+    }
+
     protected String getHandleUrl(String host, String path, ResultDelivery delivery) {
         return "http://" + host + path + handlePath(delivery);
     }
@@ -363,16 +307,23 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
     protected ResponseMetrics buildResponseFooters(long elapsedStart, long errorCount, Stats stats,
             RequestExecutionState executionState, Charset resultCharset, ResponsePrinter responsePrinter,
             ResultDelivery delivery) {
-        if (ResultDelivery.ASYNC != delivery) {
-            // in case of ASYNC delivery, the status is printed by query translator
-            responsePrinter.addFooterPrinter(new StatusPrinter(executionState.getResultStatus()));
-        }
         final ResponseMetrics metrics = ResponseMetrics.of(System.nanoTime() - elapsedStart, executionState.duration(),
                 stats.getCount(), stats.getSize(), stats.getProcessedObjects(), errorCount,
                 stats.getTotalWarningsCount(), stats.getCompileTime(), stats.getQueueWaitTime(),
                 stats.getBufferCacheHitRatio(), stats.getBufferCachePageReadCount(), stats.getCloudReadRequestsCount(),
                 stats.getCloudPagesReadCount(), stats.getCloudPagesPersistedCount());
-        responsePrinter.addFooterPrinter(new MetricsPrinter(metrics, resultCharset));
+        if (ResultDelivery.ASYNC != delivery) {
+            // in case of ASYNC delivery, the status is printed by query translator
+            responsePrinter.addFooterPrinter(new StatusPrinter(executionState.getResultStatus()));
+            responsePrinter.addFooterPrinter(new MetricsPrinter(metrics, resultCharset));
+        } else {
+            // Only print selected metrics for async requests
+            responsePrinter.addFooterPrinter(new MetricsPrinter(metrics, resultCharset,
+                    Set.of(MetricsPrinter.Metrics.ELAPSED_TIME, MetricsPrinter.Metrics.QUEUE_WAIT_TIME,
+                            MetricsPrinter.Metrics.COMPILE_TIME, MetricsPrinter.Metrics.WARNING_COUNT,
+                            MetricsPrinter.Metrics.ERROR_COUNT)));
+        }
+
         if (isPrintingProfile(stats)) {
             responsePrinter.addFooterPrinter(new ProfilePrinter(stats.getJobProfile()));
         }
@@ -436,63 +387,10 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
         buildResponseResults(responsePrinter, sessionOutput, translator.getExecutionPlans(), warnings, executionState);
     }
 
-    protected boolean handleIFormattedException(IError error, IFormattedException ex,
-            RequestExecutionState executionState, QueryServiceRequestParameters param, IServletResponse response) {
-        if (error instanceof ErrorCode) {
-            switch ((ErrorCode) error) {
-                case INVALID_REQ_PARAM_VAL:
-                case INVALID_REQ_JSON_VAL:
-                case NO_STATEMENT_PROVIDED:
-                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
-                    return true;
-                case REQUEST_TIMEOUT:
-                    logException(Level.INFO, "request execution timed out", param.getRequestId(),
-                            param.getClientContextID());
-                    executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
-                    return true;
-                case REJECT_NODE_UNREGISTERED:
-                case REJECT_BAD_CLUSTER_STATE:
-                    logException(Level.WARN, ex.getMessage(), param.getRequestId(), param.getClientContextID());
-                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
-                    return true;
-                default:
-                    // fall-through
-            }
-        } else if (error instanceof org.apache.hyracks.api.exceptions.ErrorCode) {
-            switch ((org.apache.hyracks.api.exceptions.ErrorCode) error) {
-                case JOB_REQUIREMENTS_EXCEED_CAPACITY:
-                    executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
-                    return true;
-            }
-        }
-        return false;
-    }
-
-    protected void handleExecuteStatementException(Throwable t, RequestExecutionState executionState,
-            QueryServiceRequestParameters param, IServletResponse response) {
-        if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) {
-            if (LOGGER.isDebugEnabled()) {
-                logException(Level.DEBUG, t.getMessage(), param.getRequestId(), param.getClientContextID(), t);
-            } else {
-                logException(Level.INFO, t.getMessage(), param.getRequestId(), param.getClientContextID());
-            }
-            executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
-            return;
-        } else if (t instanceof IFormattedException) {
-            IFormattedException formattedEx = (IFormattedException) t;
-            Optional<IError> maybeError = formattedEx.getError();
-            if (maybeError.isPresent() && handleIFormattedException(maybeError.get(), (IFormattedException) t,
-                    executionState, param, response)) {
-                return;
-            }
-        }
-        logException(Level.WARN, "unexpected exception", param.getRequestId(), param.getClientContextID(), t);
-        executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
-    }
-
     private void setSessionConfig(SessionOutput sessionOutput, QueryServiceRequestParameters param,
             ResultDelivery delivery) {
-        String handleUrl = getHandleUrl(param.getHost(), param.getPath(), delivery);
+        String handleUrl = param.isIncludeHost() ? getHandleUrl(param.getHost(), param.getPath(), delivery)
+                : getHandleUrl(param.getPath(), delivery);
         sessionOutput.setHandleAppender(ResultUtil.createResultHandleAppender(handleUrl));
         SessionConfig sessionConfig = sessionOutput.config();
         SessionConfig.ClientType clientType = param.getClientType();
@@ -516,12 +414,8 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
                 format != SessionConfig.OutputFormat.CLEAN_JSON && format != SessionConfig.OutputFormat.LOSSLESS_JSON
                         && format != SessionConfig.OutputFormat.LOSSLESS_ADM_JSON);
         sessionConfig.set(SessionConfig.FORMAT_CSV_HEADER, param.isCSVWithHeader());
-    }
-
-    protected void requestFailed(Throwable throwable, ResponsePrinter responsePrinter,
-            RequestExecutionState executionState) {
-        final ExecutionError executionError = ExecutionError.of(throwable);
-        responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(executionError)));
+        sessionConfig.setTimeout(param.getTimeout());
+        sessionConfig.setIncludeHost(param.isIncludeHost());
     }
 
     protected QueryServiceRequestParameters newQueryRequestParameters() {
@@ -560,12 +454,4 @@ public class QueryServiceServlet extends AbstractQueryApiServlet {
     protected String getApplicationVersion() {
         return ApplicationConfigurator.getApplicationVersion(appCtx.getBuildProperties());
     }
-
-    private void logException(Level lvl, String msg, String clientCtxId, String uuid) {
-        LOGGER.log(lvl, "handleException: {}: uuid={}, clientContextID={}", msg, uuid, clientCtxId);
-    }
-
-    private void logException(Level lvl, String msg, String clientCtxId, String uuid, Throwable t) {
-        LOGGER.log(lvl, "handleException: {}: uuid={}, clientContextID={}", msg, uuid, clientCtxId, t);
-    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
index c91f0e34ea..52597ad9d9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryStatusApiServlet.java
@@ -18,27 +18,33 @@
  */
 package org.apache.asterix.api.http.server;
 
-import static org.apache.asterix.api.http.server.AbstractQueryApiServlet.ResultStatus.FAILED;
-
 import java.io.PrintWriter;
-import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.TimeoutException;
 
-import org.apache.asterix.app.result.ExecutionError;
+import org.apache.asterix.app.result.ResponseMetrics;
 import org.apache.asterix.app.result.ResponsePrinter;
 import org.apache.asterix.app.result.ResultHandle;
 import org.apache.asterix.app.result.ResultReader;
-import org.apache.asterix.app.result.fields.ErrorsPrinter;
+import org.apache.asterix.app.result.fields.CreatedAtPrinter;
+import org.apache.asterix.app.result.fields.MetricsPrinter;
+import org.apache.asterix.app.result.fields.PartitionInfoPrinter;
+import org.apache.asterix.app.result.fields.ResultCountPrinter;
 import org.apache.asterix.app.result.fields.ResultHandlePrinter;
 import org.apache.asterix.app.result.fields.StatusPrinter;
 import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
+import org.apache.asterix.translator.IStatementExecutor;
+import org.apache.asterix.translator.ResultMetadata;
 import org.apache.asterix.translator.SessionOutput;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.result.ResultJobRecord;
 import org.apache.hyracks.http.api.IServletRequest;
 import org.apache.hyracks.http.api.IServletResponse;
 import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -54,51 +60,92 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
     @Override
     protected void get(IServletRequest request, IServletResponse response) throws Exception {
         final String strHandle = localPath(request);
-        final ResultHandle handle = ResultHandle.parse(strHandle);
+        final ResultHandle handle = parseAndValidateHandle(request, response);
         if (handle == null) {
-            response.setStatus(HttpResponseStatus.BAD_REQUEST);
-            return;
+            return; // Response status already set in parseAndValidateHandle
         }
+        boolean uriMode = handle.getRequestId() != null;
         ResultReader resultReader = new ResultReader(getResultSet(), handle.getJobId(), handle.getResultSetId());
         final ResultJobRecord.Status resultReaderStatus = resultReader.getStatus();
         if (resultReaderStatus == null) {
-            LOGGER.log(Level.INFO, "No results for: \"" + strHandle + "\"");
+            LOGGER.info("No results for: \"{}\"", strHandle);
             response.setStatus(HttpResponseStatus.NOT_FOUND);
             return;
         }
         ResultStatus resultStatus = resultStatus(resultReaderStatus);
-        Exception ex = extractException(resultReaderStatus);
+        RequestExecutionState executionState = new RequestExecutionState();
         HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
-        final PrintWriter resultWriter = response.writer();
+        // Always return 200 OK for valid status requests irrespective of the result status.
         response.setStatus(HttpResponseStatus.OK);
+        final PrintWriter resultWriter = response.writer();
         SessionOutput sessionOutput = new SessionOutput(resultWriter);
         ResponsePrinter printer = new ResponsePrinter(sessionOutput);
         printer.begin();
         printer.addHeaderPrinter(new StatusPrinter(resultStatus));
         printer.printHeaders();
-        if (ResultStatus.SUCCESS == resultStatus) {
-            String servletPath = servletPath(request).replace("status", "result");
-            String resHandle = "http://" + host(request) + servletPath + strHandle;
-            printer.addResultPrinter(new ResultHandlePrinter(resHandle));
-        } else if (ex != null) {
-            printer.addResultPrinter(new ErrorsPrinter(Collections.singletonList(ExecutionError.of(ex))));
+        switch (resultStatus) {
+            case SUCCESS -> handleSuccessfulResult(request, strHandle, uriMode, printer, resultReader);
+            case TIMEOUT -> handleTimeout(handle, executionState, printer, response);
+            case FATAL, FAILED -> handleFailure(handle, executionState, printer, response, resultReaderStatus);
+            case QUEUED, RUNNING -> {}
         }
         printer.printResults();
+        if (uriMode) {
+            printMetricsAndFooters(printer, resultReader, request);
+        }
         printer.end();
         if (response.writer().checkError()) {
             LOGGER.warn("Error flushing output writer");
         }
     }
 
+    private void handleSuccessfulResult(IServletRequest request, String strHandle, boolean uriMode,
+            ResponsePrinter printer, ResultReader resultReader) throws HyracksDataException {
+        String servletPath = servletPath(request).replace("status", "result");
+        String resHandle;
+        if (uriMode) {
+            resHandle = servletPath + strHandle;
+        } else {
+            resHandle = "http://" + host(request) + servletPath + strHandle;
+        }
+        printer.addResultPrinter(new ResultHandlePrinter(resHandle));
+        if (uriMode) {
+            printer.addResultPrinter(new ResultCountPrinter(
+                    ((ResultMetadata) (resultReader.getResultSetReader().getResultMetadata())).getResultCount()));
+            printer.addResultPrinter(
+                    new PartitionInfoPrinter(resultReader.getResultSetReader().getResultRecords(), resHandle));
+        }
+    }
+
+    private void handleTimeout(ResultHandle handle, RequestExecutionState executionState, ResponsePrinter printer,
+            IServletResponse response) {
+        RuntimeDataException hde = new RuntimeDataException(ErrorCode.REQUEST_TIMEOUT);
+        hde.addSuppressed(new TimeoutException());
+        handleExecuteStatementException(hde, executionState, handle.getRequestId(), null, response);
+        requestFailed(hde, printer, executionState);
+    }
+
+    private void handleFailure(ResultHandle handle, RequestExecutionState executionState, ResponsePrinter printer,
+            IServletResponse response, ResultJobRecord.Status resultReaderStatus) {
+        Exception ex = extractException(resultReaderStatus);
+        if (ex != null) {
+            handleExecuteStatementException(ex, executionState, handle.getRequestId(), null, response);
+            requestFailed(ex, printer, executionState);
+        }
+    }
+
     ResultStatus resultStatus(ResultJobRecord.Status status) {
         switch (status.getState()) {
             case IDLE:
+                return ResultStatus.QUEUED;
             case RUNNING:
                 return ResultStatus.RUNNING;
             case SUCCESS:
                 return ResultStatus.SUCCESS;
             case FAILED:
-                return FAILED;
+                return ResultStatus.FAILED;
+            case TIMEOUT:
+                return ResultStatus.TIMEOUT;
             default:
                 return ResultStatus.FATAL;
         }
@@ -116,4 +163,34 @@ public class QueryStatusApiServlet extends AbstractQueryApiServlet {
                 return null;
         }
     }
+
+    private void printMetricsAndFooters(ResponsePrinter printer, ResultReader resultReader, IServletRequest request)
+            throws HyracksDataException {
+        ResultMetadata metadata = (ResultMetadata) resultReader.getMetadata();
+        if (metadata != null) {
+            final IStatementExecutor.Stats stats = new IStatementExecutor.Stats();
+            stats.setProcessedObjects(metadata.getProcessedObjects());
+            stats.setQueueWaitTime(metadata.getQueueWaitTimeInNanos());
+            stats.setBufferCacheHitRatio(metadata.getBufferCacheHitRatio());
+            stats.setBufferCachePageReadCount(metadata.getBufferCachePageReadCount());
+            stats.setCloudReadRequestsCount(metadata.getCloudReadRequestsCount());
+            stats.setCloudPagesReadCount(metadata.getCloudPagesReadCount());
+            stats.setCloudPagesPersistedCount(metadata.getCloudPagesPersistedCount());
+            stats.updateTotalWarningsCount(metadata.getTotalWarningsCount());
+            long endTime = System.currentTimeMillis();
+            ResponseMetrics metrics =
+                    ResponseMetrics.of(endTime - metadata.getCreateTime(), metadata.getJobDuration(), stats.getCount(),
+                            stats.getSize(), metadata.getProcessedObjects(), 0, metadata.getTotalWarningsCount(),
+                            metadata.getCompileTime(), stats.getQueueWaitTime(), stats.getBufferCacheHitRatio(),
+                            stats.getBufferCachePageReadCount(), stats.getCloudReadRequestsCount(),
+                            stats.getCloudPagesReadCount(), stats.getCloudPagesPersistedCount());
+            printer.addFooterPrinter(new MetricsPrinter(metrics, HttpUtil.getPreferredCharset(request),
+                    Set.of(MetricsPrinter.Metrics.ELAPSED_TIME, MetricsPrinter.Metrics.EXECUTION_TIME,
+                            MetricsPrinter.Metrics.QUEUE_WAIT_TIME, MetricsPrinter.Metrics.COMPILE_TIME,
+                            MetricsPrinter.Metrics.WARNING_COUNT, MetricsPrinter.Metrics.ERROR_COUNT,
+                            MetricsPrinter.Metrics.PROCESSED_OBJECTS_COUNT)));
+            printer.addFooterPrinter(new CreatedAtPrinter(metadata.getCreateTime()));
+        }
+        printer.printFooters();
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java
index 3b1b2e8bd0..a3d9095e7a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ResultUtil.java
@@ -52,6 +52,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.prettyprint.AlgebricksAppendable;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.IFormattedException;
+import org.apache.hyracks.api.result.ResultJobRecord;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -59,6 +60,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 
+import io.netty.handler.codec.http.HttpResponseStatus;
+
 public class ResultUtil {
     private static final Logger LOGGER = LogManager.getLogger();
     public static final List<Pair<Character, String>> HTML_ENTITIES =
@@ -324,6 +327,24 @@ public class ResultUtil {
                 .append("\"");
     }
 
+    public static HttpResponseStatus getHttpStatusFromResultStatus(ResultJobRecord.Status status) {
+        if (status == null) {
+            return HttpResponseStatus.NOT_FOUND;
+        }
+        switch (status.getState()) {
+            case SUCCESS:
+                return HttpResponseStatus.OK;
+            case RUNNING:
+            case IDLE:
+            case FAILED:
+            case REMOVED:
+            case TIMEOUT:
+                return HttpResponseStatus.NOT_FOUND;
+            default:
+                return HttpResponseStatus.INTERNAL_SERVER_ERROR;
+        }
+    }
+
     public static class ParseOnlyResult {
 
         private final Set<VariableExpr> externalVariables;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoRequestMessage.java
new file mode 100644
index 0000000000..c244aa0f56
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoRequestMessage.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import java.util.Optional;
+
+import org.apache.asterix.common.api.IClientRequest;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.messaging.CCMessageBroker;
+import org.apache.asterix.translator.ClientRequest;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class ClientInfoRequestMessage implements ICcAddressedMessage {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final long serialVersionUID = 1L;
+    private final String nodeId;
+    private final JobId jobId;
+    private final String requestId;
+    private final long ncReqId;
+
+    public ClientInfoRequestMessage(String nodeId, long ncReqId, JobId jobId, String requestId) {
+        this.nodeId = nodeId;
+        this.ncReqId = ncReqId;
+        this.jobId = jobId;
+        this.requestId = requestId;
+    }
+
+    @Override
+    public void handle(ICcApplicationContext appCtx) throws HyracksDataException {
+        CCMessageBroker messageBroker = (CCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+        Optional<IClientRequest> clientRequest = appCtx.getRequestTracker().getAsyncOrDeferredRequest(requestId);
+        ClientInfoResponseMessage response;
+        if (clientRequest.isEmpty()) {
+            response = new ClientInfoResponseMessage(ncReqId, false);
+        } else {
+            response = new ClientInfoResponseMessage(ncReqId,
+                    jobId.equals(((ClientRequest) clientRequest.get()).getJobId()));
+        }
+        try {
+            messageBroker.sendApplicationMessageToNC(response, nodeId);
+        } catch (Exception e) {
+            LOGGER.info("Failed to process request", e);
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoResponseMessage.java
new file mode 100644
index 0000000000..9952a9aba0
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ClientInfoResponseMessage.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.asterix.messaging.NCMessageBroker;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class ClientInfoResponseMessage implements INcAddressedMessage {
+
+    private static final long serialVersionUID = 1L;
+    private final long reqId;
+    private final boolean validRequestId;
+
+    public ClientInfoResponseMessage(long reqId, boolean validRequestId) {
+        this.reqId = reqId;
+        this.validRequestId = validRequestId;
+    }
+
+    @Override
+    public void handle(INcApplicationContext appCtx) throws HyracksDataException, InterruptedException {
+        NCMessageBroker mb = (NCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+        MessageFuture future = mb.deregisterMessageFuture(reqId);
+        if (future != null) {
+            future.complete(this);
+        }
+    }
+
+    public boolean isValidRequestId() {
+        return validRequestId;
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultPartitionRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultPartitionRequestMessage.java
new file mode 100644
index 0000000000..f55dcb543e
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultPartitionRequestMessage.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.messaging.CcIdentifiedMessage;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultPartitionManager;
+import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class DiscardResultPartitionRequestMessage extends CcIdentifiedMessage implements INcAddressedMessage {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final long serialVersionUID = 1L;
+    private final JobId jobId;
+
+    public DiscardResultPartitionRequestMessage(JobId jobId) {
+        this.jobId = jobId;
+    }
+
+    @Override
+    public void handle(INcApplicationContext appCtx) throws HyracksDataException {
+        try {
+            IResultPartitionManager resultPartitionManager =
+                    ((NodeControllerService) appCtx.getServiceContext().getControllerService())
+                            .getResultPartitionManager();
+            resultPartitionManager.sweep(jobId);
+        } catch (Exception e) {
+            LOGGER.info("failed to remove result set", e);
+        }
+    }
+
+    @Override
+    public boolean isWhispered() {
+        return true;
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultRequestMessage.java
new file mode 100644
index 0000000000..b7486c5acd
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultRequestMessage.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.messaging.CCMessageBroker;
+import org.apache.asterix.utils.AsyncRequestsAPIUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+public class DiscardResultRequestMessage implements ICcAddressedMessage {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final long serialVersionUID = 1L;
+    private final String nodeId;
+    private final JobId jobId;
+    private final ResultSetId resultSetId;
+    private final String requestId;
+    private final long ncReqId;
+
+    public DiscardResultRequestMessage(String nodeId, long ncReqId, JobId jobId, ResultSetId resultSetId,
+            String requestId) {
+        this.nodeId = nodeId;
+        this.ncReqId = ncReqId;
+        this.jobId = jobId;
+        this.resultSetId = resultSetId;
+        this.requestId = requestId;
+    }
+
+    @Override
+    public void handle(ICcApplicationContext appCtx) throws HyracksDataException {
+        CCMessageBroker messageBroker = (CCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+        AsyncRequestsAPIUtil.discardResultPartitions((ICcApplicationContext) appCtx, jobId, resultSetId, requestId);
+        DiscardResultResponseMessage response = new DiscardResultResponseMessage(this.ncReqId);
+        try {
+            messageBroker.sendApplicationMessageToNC(response, nodeId);
+        } catch (Exception e) {
+            LOGGER.info("Failed to process request", e);
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultResponseMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultResponseMessage.java
new file mode 100644
index 0000000000..d938517b8b
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/DiscardResultResponseMessage.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.message;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.asterix.messaging.NCMessageBroker;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class DiscardResultResponseMessage implements INcAddressedMessage {
+
+    private static final long serialVersionUID = 1L;
+    private final long reqId;
+
+    public DiscardResultResponseMessage(long reqId) {
+        this.reqId = reqId;
+    }
+
+    @Override
+    public void handle(INcApplicationContext appCtx) throws HyracksDataException, InterruptedException {
+        NCMessageBroker mb = (NCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+        MessageFuture future = mb.deregisterMessageFuture(reqId);
+        if (future != null) {
+            future.complete(this);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
index b5a2afe846..a941100c6f 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/JobResultCallback.java
@@ -66,6 +66,7 @@ public class JobResultCallback implements IJobResultCallback {
         }
         final ResultMetadata metadata = (ResultMetadata) resultSetMetaData.getMetadata();
         metadata.setJobDuration(resultJobRecord.getJobDuration());
+        metadata.setResultCount(resultJobRecord.getResultCount());
         aggregateJobStats(jobId, metadata);
     }
 
@@ -105,6 +106,8 @@ public class JobResultCallback implements IJobResultCallback {
                     }
                 }
             }
+            metadata.setEndTime(System.currentTimeMillis());
+            metadata.setCreateTime(run.getCreateTime());
             metadata.setQueueWaitTimeInNanos(TimeUnit.MILLISECONDS.toNanos(run.getQueueWaitTimeInMillis()));
         }
         metadata.setProcessedObjects(processedObjects);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
index 330cf4b0f3..5938a8c611 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultHandle.java
@@ -24,29 +24,68 @@ import org.apache.hyracks.api.result.ResultSetId;
 public class ResultHandle {
     private final JobId jobId;
     private final ResultSetId resultSetId;
+    private final String requestId;
+    private final int partition;
 
-    public ResultHandle(JobId jobId, ResultSetId resultSetId) {
+    public ResultHandle(JobId jobId, ResultSetId resultSetId, String requestId, int partition) {
         this.jobId = jobId;
         this.resultSetId = resultSetId;
+        this.requestId = requestId;
+        this.partition = partition;
     }
 
-    public ResultHandle(long jobId, long resultSetId) {
-        this(new JobId(jobId), new ResultSetId(resultSetId));
+    public ResultHandle(JobId jobId, ResultSetId resultSetId, String requestId) {
+        this(jobId, resultSetId, requestId, -1);
+    }
+
+    public ResultHandle(long jobId, long resultSetId, String requestId, int partition) {
+        this(new JobId(jobId), new ResultSetId(resultSetId), requestId, partition);
     }
 
     public static ResultHandle parse(String str) {
-        int dash = str.indexOf('-');
-        if (dash < 1) {
-            return null;
-        }
         int start = 0;
-        while (str.charAt(start) == '/') {
+        while (start < str.length() && str.charAt(start) == '/') {
             ++start;
         }
-        String jobIdStr = str.substring(start, dash);
-        String resIdStr = str.substring(dash + 1);
+        str = str.substring(start);
+        String[] pathParts = str.split("/");
+
+        switch (pathParts.length) {
+            case 1:
+                // Format: jobId-resultSetId
+                return parseJobAndResultId(pathParts[0], null, -1);
+            case 2:
+                // Format: requestId/jobId-resultSetId
+                return parseJobAndResultId(pathParts[1], pathParts[0], -1);
+            case 3:
+                // Format: requestId/jobId-resultSetId/partition
+                try {
+                    int partition = Integer.parseInt(pathParts[2]);
+                    if (partition < 0) {
+                        return null;
+                    }
+                    return parseJobAndResultId(pathParts[1], pathParts[0], partition);
+                } catch (NumberFormatException e) {
+                    return null;
+                }
+            default:
+                return null;
+        }
+    }
+
+    private static ResultHandle parseJobAndResultId(String jobResPart, String requestId, int partition) {
+        int dash = jobResPart.indexOf('-');
+        if (dash < 1) {
+            return null;
+        }
+
+        String jobIdStr = jobResPart.substring(0, dash);
+        String resIdStr = jobResPart.substring(dash + 1);
+
         try {
-            return new ResultHandle(Long.parseLong(jobIdStr), Long.parseLong(resIdStr));
+            long jobId = Long.parseLong(jobIdStr);
+            long resultSetId = Long.parseLong(resIdStr);
+            return new ResultHandle(jobId, resultSetId, requestId, partition);
         } catch (NumberFormatException e) {
             return null;
         }
@@ -60,8 +99,20 @@ public class ResultHandle {
         return resultSetId;
     }
 
+    public String getRequestId() {
+        return requestId;
+    }
+
+    public int getPartition() {
+        return partition;
+    }
+
     @Override
     public String toString() {
-        return jobId.getId() + "-" + resultSetId.getId();
+        if (requestId == null) {
+            return jobId.getId() + "-" + resultSetId.getId();
+        } else {
+            return requestId + "/" + jobId.getId() + "-" + resultSetId.getId();
+        }
     }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
index 56dfd5e0c2..bd5206b651 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/ResultReader.java
@@ -46,6 +46,12 @@ public class ResultReader {
         frameTupleAccessor = new ResultFrameTupleAccessor();
     }
 
+    public ResultReader(IResultSet resultSet, JobId jobId, ResultSetId resultSetId, int partition)
+            throws HyracksDataException {
+        reader = resultSet.createReader(jobId, resultSetId, partition);
+        frameTupleAccessor = new ResultFrameTupleAccessor();
+    }
+
     public Status getStatus() {
         return reader.getResultStatus();
     }
@@ -67,4 +73,8 @@ public class ResultReader {
         return new ResultSet(hcc, srv.getNetworkSecurityManager().getSocketChannelFactory(),
                 compilerProperties.getFrameSize(), ResultReader.NUM_READERS);
     }
+
+    public IResultSetReader getResultSetReader() {
+        return reader;
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/CreatedAtPrinter.java
similarity index 51%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/CreatedAtPrinter.java
index 1652dd42f4..d7192f5c00 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/CreatedAtPrinter.java
@@ -16,21 +16,31 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.api.result;
+package org.apache.asterix.app.result.fields;
 
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import java.io.PrintWriter;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
 
-public interface IResultSetReader {
-    Status getResultStatus();
+import org.apache.asterix.common.api.IResponseFieldPrinter;
 
-    int read(IFrame frame) throws HyracksDataException;
+public class CreatedAtPrinter implements IResponseFieldPrinter {
 
-    /**
-     * Gets the result metadata
-     *
-     * @return the result metadata
-     */
-    IResultMetadata getResultMetadata();
+    public static final String FIELD_NAME = "createdAt";
+    private final long createdAt;
+
+    public CreatedAtPrinter(long createdAt) {
+        this.createdAt = createdAt;
+    }
+
+    @Override
+    public void print(PrintWriter pw) {
+        pw.append("\t\"").append(FIELD_NAME).append("\": \"")
+                .append(DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(createdAt))).append("\"");
+    }
+
+    @Override
+    public String getName() {
+        return FIELD_NAME;
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
index 1e6553334c..952c897a7c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/MetricsPrinter.java
@@ -21,6 +21,9 @@ package org.apache.asterix.app.result.fields;
 import java.io.PrintWriter;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
 
 import org.apache.asterix.api.http.server.ResultUtil;
 import org.apache.asterix.app.result.ResponseMetrics;
@@ -59,78 +62,132 @@ public class MetricsPrinter implements IResponseFieldPrinter {
     public static final String FIELD_NAME = "metrics";
     private final ResponseMetrics metrics;
     private final Charset resultCharset;
+    private final Set<Metrics> selectedMetrics;
 
     public MetricsPrinter(ResponseMetrics metrics, Charset resultCharset) {
+        this(metrics, resultCharset, null);
+    }
+
+    public MetricsPrinter(ResponseMetrics metrics, Charset resultCharset, Set<Metrics> selectedMetrics) {
         this.metrics = metrics;
         this.resultCharset = resultCharset;
+        this.selectedMetrics = selectedMetrics == null ? Set.of(Metrics.values()) : selectedMetrics;
     }
 
     @Override
     public void print(PrintWriter pw) {
         boolean useAscii = !StandardCharsets.UTF_8.equals(resultCharset)
                 && !"μ".contentEquals(resultCharset.decode(resultCharset.encode("μ")));
-        pw.print("\t\"");
-        pw.print(FIELD_NAME);
-        pw.print("\": {\n");
-        pw.print("\t");
-        ResultUtil.printField(pw, Metrics.ELAPSED_TIME.str(), Duration.formatNanos(metrics.getElapsedTime(), useAscii));
-        pw.print("\n\t");
-        ResultUtil.printField(pw, Metrics.EXECUTION_TIME.str(),
-                Duration.formatNanos(metrics.getExecutionTime(), useAscii));
-        pw.print("\n\t");
-        ResultUtil.printField(pw, Metrics.COMPILE_TIME.str(), Duration.formatNanos(metrics.getCompileTime(), useAscii));
-        pw.print("\n\t");
-        ResultUtil.printField(pw, Metrics.QUEUE_WAIT_TIME.str(),
-                Duration.formatNanos(metrics.getQueueWaitTime(), useAscii));
-        pw.print("\n\t");
-        ResultUtil.printField(pw, Metrics.RESULT_COUNT.str(), metrics.getResultCount(), true);
-        pw.print("\n\t");
-        ResultUtil.printField(pw, Metrics.RESULT_SIZE.str(), metrics.getResultSize(), true);
-        pw.print("\n\t");
+
         final boolean hasErrors = metrics.getErrorCount() > 0;
         final boolean hasWarnings = metrics.getWarnCount() > 0;
         final boolean usedCache = !(Double.isNaN(metrics.getBufferCacheHitRatio()));
         final boolean madeCloudReadRequests = metrics.getCloudReadRequestsCount() > 0;
-        ResultUtil.printField(pw, Metrics.PROCESSED_OBJECTS_COUNT.str(), metrics.getProcessedObjects(),
-                usedCache || hasWarnings || hasErrors);
-        pw.print("\n");
+
+        // Entry representation with typed value
+        abstract class Entry {
+            final Metrics m;
+
+            Entry(Metrics m) {
+                this.m = m;
+            }
+
+            abstract void print(PrintWriter out, boolean more);
+        }
+        class StringEntry extends Entry {
+            final String v;
+
+            StringEntry(Metrics m, String v) {
+                super(m);
+                this.v = v;
+            }
+
+            @Override
+            void print(PrintWriter out, boolean more) {
+                ResultUtil.printField(out, m.str(), v, more);
+            }
+        }
+        class LongEntry extends Entry {
+            final long v;
+
+            LongEntry(Metrics m, long v) {
+                super(m);
+                this.v = v;
+            }
+
+            @Override
+            void print(PrintWriter out, boolean more) {
+                ResultUtil.printField(out, m.str(), v, more);
+            }
+        }
+
+        List<Entry> entries = new ArrayList<>();
+
+        if (isSelected(Metrics.ELAPSED_TIME))
+            entries.add(
+                    new StringEntry(Metrics.ELAPSED_TIME, Duration.formatNanos(metrics.getElapsedTime(), useAscii)));
+        if (isSelected(Metrics.EXECUTION_TIME))
+            entries.add(new StringEntry(Metrics.EXECUTION_TIME,
+                    Duration.formatNanos(metrics.getExecutionTime(), useAscii)));
+        if (isSelected(Metrics.COMPILE_TIME))
+            entries.add(
+                    new StringEntry(Metrics.COMPILE_TIME, Duration.formatNanos(metrics.getCompileTime(), useAscii)));
+        if (isSelected(Metrics.QUEUE_WAIT_TIME))
+            entries.add(new StringEntry(Metrics.QUEUE_WAIT_TIME,
+                    Duration.formatNanos(metrics.getQueueWaitTime(), useAscii)));
+        if (isSelected(Metrics.RESULT_COUNT))
+            entries.add(new LongEntry(Metrics.RESULT_COUNT, metrics.getResultCount()));
+        if (isSelected(Metrics.RESULT_SIZE))
+            entries.add(new LongEntry(Metrics.RESULT_SIZE, metrics.getResultSize()));
+        if (isSelected(Metrics.PROCESSED_OBJECTS_COUNT))
+            entries.add(new LongEntry(Metrics.PROCESSED_OBJECTS_COUNT, metrics.getProcessedObjects()));
+
         if (usedCache) {
-            pw.print("\t");
-            String pctValue = String.format("%.2f%%", metrics.getBufferCacheHitRatio() * 100);
-            ResultUtil.printField(pw, Metrics.BUFFERCACHE_HIT_RATIO.str(), pctValue, true);
-            pw.print("\n");
-            pw.print("\t");
-            ResultUtil.printField(pw, Metrics.BUFFERCACHE_PAGEREAD_COUNT.str(), metrics.getBufferCachePageReadCount(),
-                    hasWarnings || hasErrors || madeCloudReadRequests);
-            pw.print("\n");
+            if (isSelected(Metrics.BUFFERCACHE_HIT_RATIO)) {
+                String pctValue = String.format("%.2f%%", metrics.getBufferCacheHitRatio() * 100);
+                entries.add(new StringEntry(Metrics.BUFFERCACHE_HIT_RATIO, pctValue));
+            }
+            if (isSelected(Metrics.BUFFERCACHE_PAGEREAD_COUNT)) {
+                entries.add(new LongEntry(Metrics.BUFFERCACHE_PAGEREAD_COUNT, metrics.getBufferCachePageReadCount()));
+            }
         }
+
         if (madeCloudReadRequests) {
-            pw.print("\t");
-            ResultUtil.printField(pw, Metrics.REMOTE_STORAGE_REQUESTS_COUNT.str(), metrics.getCloudReadRequestsCount(),
-                    true);
-            pw.print("\n");
-            pw.print("\t");
-            ResultUtil.printField(pw, Metrics.REMOTE_STORAGE_PAGES_READ_COUNT.str(), metrics.getCloudPagesReadCount(),
-                    true);
-            pw.print("\n");
-            pw.print("\t");
-            ResultUtil.printField(pw, Metrics.REMOTE_PAGES_PERSISTED_COUNT.str(), metrics.getCloudPagesPersistedCount(),
-                    hasWarnings || hasErrors);
-            pw.print("\n");
+            if (isSelected(Metrics.REMOTE_STORAGE_REQUESTS_COUNT)) {
+                entries.add(new LongEntry(Metrics.REMOTE_STORAGE_REQUESTS_COUNT, metrics.getCloudReadRequestsCount()));
+            }
+            if (isSelected(Metrics.REMOTE_STORAGE_PAGES_READ_COUNT)) {
+                entries.add(new LongEntry(Metrics.REMOTE_STORAGE_PAGES_READ_COUNT, metrics.getCloudPagesReadCount()));
+            }
+            if (isSelected(Metrics.REMOTE_PAGES_PERSISTED_COUNT)) {
+                entries.add(new LongEntry(Metrics.REMOTE_PAGES_PERSISTED_COUNT, metrics.getCloudPagesPersistedCount()));
+            }
         }
-        if (hasWarnings) {
-            pw.print("\t");
-            ResultUtil.printField(pw, Metrics.WARNING_COUNT.str(), metrics.getWarnCount(), hasErrors);
-            pw.print("\n");
+
+        if (hasWarnings && isSelected(Metrics.WARNING_COUNT)) {
+            entries.add(new LongEntry(Metrics.WARNING_COUNT, metrics.getWarnCount()));
         }
-        if (hasErrors) {
+        if (hasErrors && isSelected(Metrics.ERROR_COUNT)) {
+            entries.add(new LongEntry(Metrics.ERROR_COUNT, metrics.getErrorCount()));
+        }
+
+        pw.print("\t\"");
+        pw.print(FIELD_NAME);
+        pw.print("\": {\n");
+        for (int i = 0; i < entries.size(); i++) {
+            Entry e = entries.get(i);
+            boolean hasMore = i < entries.size() - 1;
             pw.print("\t");
-            ResultUtil.printField(pw, Metrics.ERROR_COUNT.str(), metrics.getErrorCount(), false);
+            e.print(pw, hasMore);
             pw.print("\n");
         }
         pw.print("\t}");
     }
 
+    private boolean isSelected(Metrics m) {
+        return selectedMetrics.contains(m);
+    }
+
     @Override
     public String getName() {
         return FIELD_NAME;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/NcResultPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/NcResultPrinter.java
index d98472e2c2..02ef6537a0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/NcResultPrinter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/NcResultPrinter.java
@@ -26,14 +26,10 @@ import org.apache.asterix.app.message.ExecuteStatementResponseMessage;
 import org.apache.asterix.app.result.ResponsePrinter;
 import org.apache.asterix.app.result.ResultReader;
 import org.apache.asterix.common.api.IApplicationContext;
-import org.apache.asterix.om.types.ARecordType;
 import org.apache.asterix.translator.IStatementExecutor;
 import org.apache.asterix.translator.SessionOutput;
-import org.apache.commons.lang3.tuple.Triple;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.job.JobId;
 import org.apache.hyracks.api.result.IResultSet;
-import org.apache.hyracks.api.result.ResultSetId;
 
 public class NcResultPrinter extends AbstractResultsPrinter {
 
@@ -53,12 +49,12 @@ public class NcResultPrinter extends AbstractResultsPrinter {
     @Override
     public void print(PrintWriter pw) throws HyracksDataException {
         IStatementExecutor.ResultMetadata resultMetadata = responseMsg.getMetadata();
-        List<Triple<JobId, ResultSetId, ARecordType>> resultSets = resultMetadata.getResultSets();
+        List<IStatementExecutor.ResultSetInfo> resultSets = resultMetadata.getResultSets();
         if (delivery == IStatementExecutor.ResultDelivery.IMMEDIATE && !resultSets.isEmpty()) {
             for (int i = 0; i < resultSets.size(); i++) {
-                Triple<JobId, ResultSetId, ARecordType> rsmd = resultSets.get(i);
-                ResultReader resultReader = new ResultReader(resultSet, rsmd.getLeft(), rsmd.getMiddle());
-                ResultUtil.printResults(appCtx, resultReader, sessionOutput, stats, rsmd.getRight());
+                IStatementExecutor.ResultSetInfo rsmd = resultSets.get(i);
+                ResultReader resultReader = new ResultReader(resultSet, rsmd.getJobId(), rsmd.getResultSetId());
+                ResultUtil.printResults(appCtx, resultReader, sessionOutput, stats, rsmd.getRecordType());
                 if (i + 1 != resultSets.size()) {
                     ResponsePrinter.printFieldSeparator(pw);
                 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/PartitionInfoPrinter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/PartitionInfoPrinter.java
new file mode 100644
index 0000000000..8e8d7f2aa7
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/PartitionInfoPrinter.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.result.fields;
+
+import java.io.PrintWriter;
+
+import org.apache.asterix.api.http.server.ResultUtil;
+import org.apache.asterix.common.api.IResponseFieldPrinter;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+
+public class PartitionInfoPrinter implements IResponseFieldPrinter {
+    public static final String FIELD_NAME = "partitions";
+    public static final String HANDLE_FIELD_NAME = "handle";
+    public static final String RESULT_COUNT_FIELD_NAME = "resultCount";
+
+    private final ResultDirectoryRecord[] resultRecords;
+    private final String handlePrefix;
+
+    public PartitionInfoPrinter(ResultDirectoryRecord[] resultRecords, String handlePrefix) {
+        this.resultRecords = resultRecords;
+        this.handlePrefix = handlePrefix;
+    }
+
+    @Override
+    public void print(PrintWriter pw) throws HyracksDataException {
+        pw.print("\t\"");
+        pw.print(getName());
+        pw.print("\": [");
+        for (int i = 0; i < resultRecords.length; i++) {
+            final ResultDirectoryRecord record = resultRecords[i];
+            pw.print("{ \n\t");
+            ResultUtil.printField(pw, HANDLE_FIELD_NAME, handlePrefix + "/" + i);
+            pw.print("\n\t");
+            ResultUtil.printField(pw, RESULT_COUNT_FIELD_NAME, record.getResultCount(), false);
+            pw.print("\n\t} \n\t");
+            if (i < resultRecords.length - 1) {
+                pw.print(",");
+            }
+        }
+        pw.print("]");
+    }
+
+    @Override
+    public String getName() {
+        return FIELD_NAME;
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/ResultCountPrinter.java
similarity index 55%
copy from hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
copy to asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/ResultCountPrinter.java
index 1652dd42f4..241ee9a076 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/result/fields/ResultCountPrinter.java
@@ -16,21 +16,30 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-package org.apache.hyracks.api.result;
+package org.apache.asterix.app.result.fields;
 
-import org.apache.hyracks.api.comm.IFrame;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.result.ResultJobRecord.Status;
+import java.io.PrintWriter;
 
-public interface IResultSetReader {
-    Status getResultStatus();
+import org.apache.asterix.api.http.server.ResultUtil;
+import org.apache.asterix.common.api.IResponseFieldPrinter;
 
-    int read(IFrame frame) throws HyracksDataException;
+public class ResultCountPrinter implements IResponseFieldPrinter {
 
-    /**
-     * Gets the result metadata
-     *
-     * @return the result metadata
-     */
-    IResultMetadata getResultMetadata();
+    private static final String FIELD_NAME = "resultCount";
+
+    private final long resultCount;
+
+    public ResultCountPrinter(long resultCount) {
+        this.resultCount = resultCount;
+    }
+
+    @Override
+    public void print(PrintWriter pw) {
+        ResultUtil.printField(pw, FIELD_NAME, resultCount, false);
+    }
+
+    @Override
+    public String getName() {
+        return FIELD_NAME;
+    }
 }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 7f34bcb4dd..5e4706c8a8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -22,6 +22,7 @@ import static org.apache.asterix.common.api.IIdentifierMapper.Modifier.PLURAL;
 import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
 import static org.apache.asterix.common.utils.IdentifierUtil.dataverse;
 import static org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement.FIELD_TYPE_STOPWORDS;
+import static org.apache.hyracks.control.nc.result.ResultState.UNLIMITED_READS;
 
 import java.io.FileInputStream;
 import java.io.InputStream;
@@ -40,7 +41,12 @@ import java.util.Objects;
 import java.util.Optional;
 import java.util.Properties;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.stream.Collectors;
 
@@ -553,7 +559,8 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
                         metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter.getAndInc()));
                         metadataProvider.setResultAsyncMode(
                                 resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.DEFERRED);
-                        metadataProvider.setMaxResultReads(maxResultReads);
+                        metadataProvider
+                                .setMaxResultReads(!sessionConfig.isIncludeHost() ? UNLIMITED_READS : maxResultReads);
                         if (stats.getProfileType() == Stats.ProfileType.FULL) {
                             this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
                         }
@@ -4534,11 +4541,10 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         return jobId;
     }
 
-    @Override
     public JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector,
             MetadataProvider metadataProvider, Query query, ICompiledDmlStatement stmt,
-            Map<String, IAObject> stmtParams, IRequestParameters requestParameters)
-            throws AlgebricksException, ACIDException {
+            Map<String, IAObject> stmtParams, IRequestParameters requestParameters,
+            org.apache.asterix.translator.ResultMetadata resultMetadata) throws AlgebricksException, ACIDException {
 
         Map<VarIdentifier, IAObject> externalVars = createExternalVariables(query, stmtParams);
 
@@ -4551,7 +4557,15 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         // Query Compilation (happens under the same ongoing metadata transaction)
         return apiFramework.compileQuery(clusterInfoCollector, metadataProvider, (Query) rewrittenResult.first,
                 rewrittenResult.second, stmt == null ? null : stmt.getDatasetName(), sessionOutput, stmt, externalVars,
-                responsePrinter, warningCollector, requestParameters, jobFlags);
+                responsePrinter, warningCollector, requestParameters, jobFlags, resultMetadata);
+    }
+
+    @Override
+    public JobSpecification rewriteCompileQuery(IClusterInfoCollector clusterInfoCollector,
+            MetadataProvider metadataProvider, Query query, ICompiledDmlStatement stmt,
+            Map<String, IAObject> stmtParams, IRequestParameters requestParameters) throws AlgebricksException {
+        return rewriteCompileQuery(clusterInfoCollector, metadataProvider, query, stmt, stmtParams, requestParameters,
+                null);
     }
 
     protected JobSpecification rewriteCompileInsertUpsert(IClusterInfoCollector clusterInfoCollector,
@@ -5506,13 +5520,16 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             boolean bActiveTxn = true;
             metadataProvider.setMetadataTxnContext(mdTxnCtx);
             try {
-                final JobSpecification jobSpec =
-                        rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams, requestParameters);
+                org.apache.asterix.translator.ResultMetadata resultMetadata =
+                        new org.apache.asterix.translator.ResultMetadata(sessionConfig.fmt());
+                final JobSpecification jobSpec = rewriteCompileQuery(hcc, metadataProvider, query, null, stmtParams,
+                        requestParameters, resultMetadata);
                 // update stats with count of compile-time warnings. needs to be adapted for multi-statement.
                 stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
                 afterCompile();
                 MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                 stats.setCompileTime(System.nanoTime() - compileStart);
+                resultMetadata.setCompileTime(stats.getCompileTime());
                 bActiveTxn = false;
                 return query.isExplain() || isCompileOnly() ? null : jobSpec;
             } catch (Exception e) {
@@ -5555,12 +5572,17 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             case DEFERRED:
                 createAndRunJob(hcc, jobFlags, null, compiler, locker, resultDelivery, id -> {
                     updateJobStats(id, stats, metadataProvider.getResultSetId(), clientRequest);
-                    responsePrinter.addResultPrinter(
-                            new ResultHandlePrinter(sessionOutput, new ResultHandle(id, resultSetId)));
+                    if (!sessionConfig.isIncludeHost()) {
+                        responsePrinter.addResultPrinter(new ResultHandlePrinter(sessionOutput,
+                                new ResultHandle(id, resultSetId, requestParameters.getRequestReference().getUuid())));
+                    } else {
+                        responsePrinter.addResultPrinter(
+                                new ResultHandlePrinter(sessionOutput, new ResultHandle(id, resultSetId, null)));
+                    }
                     responsePrinter.printResults();
                     if (outMetadata != null) {
-                        outMetadata.getResultSets().add(org.apache.commons.lang3.tuple.Triple.of(id, resultSetId,
-                                metadataProvider.findOutputRecordType()));
+                        outMetadata.getResultSets()
+                                .add(new ResultSetInfo(id, resultSetId, metadataProvider.findOutputRecordType()));
                     }
                 }, requestParameters, cancellable, appCtx, metadataProvider, atomicStmt);
                 break;
@@ -5596,31 +5618,39 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
             ResultDelivery resultDelivery, IRequestParameters requestParameters, boolean cancellable,
             ResultSetId resultSetId, MutableBoolean printed, MetadataProvider metadataProvider, Statement atomicStmt) {
         Mutable<JobId> jobId = new MutableObject<>(JobId.INVALID);
-        try {
-            createAndRunJob(hcc, jobFlags, jobId, compiler, locker, resultDelivery, id -> {
-                final ResultHandle handle = new ResultHandle(id, resultSetId);
-                responsePrinter.addResultPrinter(new StatusPrinter(AbstractQueryApiServlet.ResultStatus.RUNNING));
-                responsePrinter.addResultPrinter(new ResultHandlePrinter(sessionOutput, handle));
-                responsePrinter.printResults();
-                synchronized (printed) {
-                    printed.setTrue();
-                    printed.notify();
-                }
-            }, requestParameters, cancellable, appCtx, metadataProvider, atomicStmt);
-        } catch (Exception e) {
-            if (Objects.equals(JobId.INVALID, jobId.getValue())) {
-                // compilation failed
-                responsePrinter.addResultPrinter(new StatusPrinter(AbstractQueryApiServlet.ResultStatus.FAILED));
-                responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(ExecutionError.of(e))));
-                try {
+        final CompletableFuture<JobId> jobIdFuture = new CompletableFuture<>();
+        Future<?> jobSubmitFuture = executorService.submit(() -> {
+            try {
+                createAndRunJob(hcc, jobFlags, jobId, compiler, locker, resultDelivery, id -> {
+                    jobIdFuture.complete(id);
+                    final ResultHandle handle = sessionConfig.isIncludeHost() ? new ResultHandle(id, resultSetId, null)
+                            : new ResultHandle(id, resultSetId, requestParameters.getRequestReference().getUuid());
+                    responsePrinter.addResultPrinter(new StatusPrinter(AbstractQueryApiServlet.ResultStatus.QUEUED));
+                    responsePrinter.addResultPrinter(new ResultHandlePrinter(sessionOutput, handle));
                     responsePrinter.printResults();
-                } catch (HyracksDataException ex) {
-                    LOGGER.error("failed to print result", ex);
-                }
-            } else {
-                GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR,
-                        resultDelivery.name() + " job with id " + jobId.getValue() + " " + "failed", e);
+                    synchronized (printed) {
+                        printed.setTrue();
+                        printed.notify();
+                    }
+                }, requestParameters, cancellable, appCtx, metadataProvider, atomicStmt);
+            } catch (Exception e) {
+                jobIdFuture.completeExceptionally(e);
+                throw new RuntimeException(e);
             }
+        });
+        try {
+            jobSubmitFuture.get(sessionOutput.config().getTimeout(), TimeUnit.MILLISECONDS);
+        } catch (TimeoutException e) {
+            handleAsyncJobTimeout(hcc, jobIdFuture, jobId.get(), jobSubmitFuture);
+        } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+            cancelIfStarted(hcc, jobIdFuture);
+            jobSubmitFuture.cancel(true);
+        } catch (ExecutionException e) {
+            Throwable cause = e.getCause() != null ? e.getCause() : e;
+            handleAsyncJobException(cause, jobId.get(), resultDelivery);
+        } catch (Exception e) {
+            handleAsyncJobException(e, jobId.get(), resultDelivery);
         } finally {
             synchronized (printed) {
                 if (printed.isFalse()) {
@@ -5631,6 +5661,44 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
         }
     }
 
+    private void handleAsyncJobTimeout(IHyracksClientConnection hcc, CompletableFuture<JobId> jobIdFuture, JobId jobId,
+            Future<?> jobSubmitFuture) {
+        cancelIfStarted(hcc, jobIdFuture);
+        jobSubmitFuture.cancel(true);
+        final ClusterControllerService controllerService =
+                (ClusterControllerService) appCtx.getServiceContext().getControllerService();
+        controllerService.getResultDirectoryService().reportJobTimeout(jobId);
+    }
+
+    private void handleAsyncJobException(Throwable e, JobId jobId, ResultDelivery resultDelivery) {
+        if (Objects.equals(JobId.INVALID, jobId)) {
+            // compilation failed
+            responsePrinter.addResultPrinter(new StatusPrinter(AbstractQueryApiServlet.ResultStatus.FAILED));
+            responsePrinter.addResultPrinter(new ErrorsPrinter(Collections.singletonList(ExecutionError.of(e))));
+            try {
+                responsePrinter.printResults();
+            } catch (HyracksDataException ex) {
+                LOGGER.error("failed to print result", ex);
+            }
+        } else {
+            GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR,
+                    resultDelivery.name() + " job with id " + jobId + " " + "failed", e);
+        }
+    }
+
+    private void cancelIfStarted(IHyracksClientConnection hcc, CompletableFuture<JobId> jobIdFuture) {
+        if (jobIdFuture.isDone() && !jobIdFuture.isCompletedExceptionally()) {
+            try {
+                JobId jobId = jobIdFuture.getNow(null);
+                if (jobId != null) {
+                    hcc.cancelJob(jobId);
+                }
+            } catch (Exception e) {
+                LOGGER.warn("Failed to cancel timed out job", e);
+            }
+        }
+    }
+
     private void runJob(IHyracksClientConnection hcc, JobSpecification jobSpec) throws Exception {
         runJob(hcc, jobSpec, jobFlags);
     }
@@ -5834,6 +5902,22 @@ public class QueryTranslator extends AbstractLangTranslator implements IStatemen
     protected void trackRequest(IRequestParameters requestParameters) throws HyracksDataException {
         final IClientRequest clientRequest = appCtx.getReceptionist().requestReceived(requestParameters);
         this.appCtx.getRequestTracker().track(clientRequest);
+        if (shouldTrackAsSeparateRequest(requestParameters)) {
+            appCtx.getRequestTracker().trackAsyncOrDeferredRequest(clientRequest);
+        }
+    }
+
+    /**
+     * For async and deferred requests, we track them as separate requests in Request Tracker
+     * since for these requests, we need to maintain their state until their result is retrieved or
+     * expired.
+     */
+    private boolean shouldTrackAsSeparateRequest(IRequestParameters requestParameters) {
+        if (sessionConfig.isIncludeHost()) {
+            return false;
+        }
+        ResultDelivery resultDelivery = requestParameters.getResultProperties().getDelivery();
+        return resultDelivery == ResultDelivery.ASYNC || resultDelivery == ResultDelivery.DEFERRED;
     }
 
     protected void validateStatements(IRequestParameters requestParameters)
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index da606c0d4c..8775603291 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -40,12 +40,12 @@ import java.util.Map;
 
 import org.apache.asterix.algebra.base.ILangExtension;
 import org.apache.asterix.api.http.server.BasicAuthServlet;
+import org.apache.asterix.api.http.server.NCQueryResultApiServlet;
 import org.apache.asterix.api.http.server.NCQueryServiceServlet;
+import org.apache.asterix.api.http.server.NCQueryStatusApiServlet;
 import org.apache.asterix.api.http.server.NCUdfApiServlet;
 import org.apache.asterix.api.http.server.NCUdfRecoveryServlet;
 import org.apache.asterix.api.http.server.NetDiagnosticsApiServlet;
-import org.apache.asterix.api.http.server.QueryResultApiServlet;
-import org.apache.asterix.api.http.server.QueryStatusApiServlet;
 import org.apache.asterix.api.http.server.ServletConstants;
 import org.apache.asterix.api.http.server.StorageApiServlet;
 import org.apache.asterix.app.config.ConfigValidator;
@@ -269,8 +269,8 @@ public class NCApplication extends BaseNCApplication {
                 apiServer.ctx(), new NCUdfRecoveryServlet(apiServer.ctx(), new String[] { UDF_RECOVERY },
                         getApplicationContext(), apiServer.getScheme(), apiServer.getAddress().getPort()),
                 auth.getFirst(), auth.getSecond()));
-        apiServer.addServlet(new QueryStatusApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_STATUS));
-        apiServer.addServlet(new QueryResultApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_RESULT));
+        apiServer.addServlet(new NCQueryStatusApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_STATUS));
+        apiServer.addServlet(new NCQueryResultApiServlet(apiServer.ctx(), getApplicationContext(), QUERY_RESULT));
         webManager.add(apiServer);
     }
 
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/AsyncRequestsAPIUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/AsyncRequestsAPIUtil.java
new file mode 100644
index 0000000000..8884b4483a
--- /dev/null
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/AsyncRequestsAPIUtil.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.utils;
+
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import org.apache.asterix.app.message.ClientInfoRequestMessage;
+import org.apache.asterix.app.message.ClientInfoResponseMessage;
+import org.apache.asterix.app.message.DiscardResultPartitionRequestMessage;
+import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.ACIDException;
+import org.apache.asterix.common.messaging.api.ICCMessageBroker;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.MessageFuture;
+import org.apache.hyracks.api.application.INCServiceContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultJobRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+import org.apache.hyracks.control.cc.result.IResultDirectoryService;
+import org.apache.hyracks.control.common.work.IResultCallback;
+import org.apache.hyracks.http.api.IServletResponse;
+import org.apache.logging.log4j.Level;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.netty.handler.codec.http.HttpResponseStatus;
+
+public class AsyncRequestsAPIUtil {
+
+    private static final Logger LOGGER = LogManager.getLogger();
+    private static final long RESULT_PARTITIONS_FETCH_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(1);
+    public static final long NC_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(5);
+
+    /**
+     * Discards result partitions across all nodes and cleans up tracking information.
+     *
+     * @param appCtx Application context
+     * @param jobId Job identifier
+     * @param resultSetId Result set identifier
+     * @param requestId Optional request identifier for cleanup
+     * @throws HyracksDataException if discard operation fails
+     */
+    public static void discardResultPartitions(ICcApplicationContext appCtx, JobId jobId, ResultSetId resultSetId,
+            String requestId) throws HyracksDataException {
+        IResultDirectoryService resultDirectoryService =
+                ((ClusterControllerService) appCtx.getServiceContext().getControllerService())
+                        .getResultDirectoryService();;
+        // Check if result is in a valid state for discarding
+        ResultJobRecord.Status status = resultDirectoryService.getResultStatus(jobId, resultSetId);
+        if (status.getState() != ResultJobRecord.State.SUCCESS) {
+            LOGGER.log(Level.WARN, "Cannot discard result for job {}, result set {}, request {} - status is {}", jobId,
+                    resultSetId, requestId, status);
+            return;
+        }
+
+        // Send discard result messages to all nodes containing result partitions
+        Set<String> nodeIds = fetchResultNodeIds(resultDirectoryService, jobId, resultSetId);
+        ICCMessageBroker messageBroker = (ICCMessageBroker) appCtx.getServiceContext().getMessageBroker();
+        for (String nodeId : nodeIds) {
+            DiscardResultPartitionRequestMessage message = new DiscardResultPartitionRequestMessage(jobId);
+            try {
+                messageBroker.sendRealTimeApplicationMessageToNC(message, nodeId);
+            } catch (Exception e) {
+                throw new ACIDException("Failed to send discard message to node " + nodeId, e);
+            }
+        }
+
+        // Clean up result directory and request tracking
+        resultDirectoryService.sweep(jobId);
+        if (requestId != null) {
+            appCtx.getRequestTracker().removeAsyncOrDeferredRequest(requestId);
+        }
+    }
+
+    private static Set<String> fetchResultNodeIds(IResultDirectoryService rds, JobId jobId, ResultSetId resultSetId)
+            throws HyracksDataException {
+        CompletableFuture<ResultDirectoryRecord[]> future = new CompletableFuture<>();
+        rds.getResultPartitionLocations(jobId, resultSetId, null, new IResultCallback<>() {
+            @Override
+            public void setValue(ResultDirectoryRecord[] result) {
+                future.complete(result);
+            }
+
+            @Override
+            public void setException(Exception e) {
+                LOGGER.log(org.apache.logging.log4j.Level.WARN, "Failed to get result partition locations", e);
+                future.completeExceptionally(e);
+            }
+        });
+        ResultDirectoryRecord[] records;
+        try {
+            records = future.get(RESULT_PARTITIONS_FETCH_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
+        } catch (ExecutionException | TimeoutException e) {
+            throw HyracksDataException.create(e);
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw HyracksDataException.create(e);
+        }
+        return Arrays.stream(records).filter(Objects::nonNull).map(ResultDirectoryRecord::getNodeId)
+                .filter(Objects::nonNull).collect(LinkedHashSet::new, Set::add, Set::addAll);
+    }
+
+    public static boolean isValidRequest(IApplicationContext appCtx, String requestId, JobId jobId,
+            IServletResponse response) throws HyracksDataException {
+        INCServiceContext serviceCtx = (INCServiceContext) appCtx.getServiceContext();
+        INCMessageBroker messageBroker = (INCMessageBroker) serviceCtx.getMessageBroker();
+        MessageFuture messageFuture = messageBroker.registerMessageFuture();
+        long futureId = messageFuture.getFutureId();
+        ClientInfoRequestMessage clientInfoRequestMessage =
+                new ClientInfoRequestMessage(serviceCtx.getNodeId(), futureId, jobId, requestId);
+        try {
+            messageBroker.sendMessageToPrimaryCC(clientInfoRequestMessage);
+            ClientInfoResponseMessage responseMessage =
+                    (ClientInfoResponseMessage) messageFuture.get(NC_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS);
+            if (responseMessage == null || !responseMessage.isValidRequestId()) {
+                response.setStatus(HttpResponseStatus.NOT_FOUND);
+                return false;
+            }
+            return true;
+        } catch (Exception e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
index ac850524e8..903376436d 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/ResultExtractor.java
@@ -119,7 +119,8 @@ public class ResultExtractor {
         TYPE("type"),
         ERRORS("errors"),
         PLANS("plans"),
-        WARNINGS("warnings");
+        WARNINGS("warnings"),
+        CREATED_AT("createdAt");
 
         private static final Map<String, ResultField> fields = new HashMap<>();
 
@@ -183,6 +184,10 @@ public class ResultExtractor {
 
     public static String extractHandle(InputStream resultStream, Charset responseCharset) throws Exception {
         String result = IOUtils.toString(resultStream, responseCharset);
+        return extractHandle(result);
+    }
+
+    public static String extractHandle(String result) throws Exception {
         ObjectNode resultJson = OBJECT_READER.readValue(result);
         final JsonNode handle = resultJson.get("handle");
         if (handle != null) {
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 32d6e170ec..df66fb59a7 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -724,6 +724,20 @@ public class TestExecutor {
 
     // For tests where you simply want the byte-for-byte output.
     private static void writeOutputToFile(File actualFile, InputStream resultStream) throws Exception {
+        ensureParentDir(actualFile);
+        try (FileOutputStream out = new FileOutputStream(actualFile)) {
+            IOUtils.copy(resultStream, out);
+        }
+    }
+
+    private static void writeOutputToFile(File actualFile, String content) throws Exception {
+        ensureParentDir(actualFile);
+        try (FileOutputStream out = new FileOutputStream(actualFile)) {
+            IOUtils.write(content, out, UTF_8);
+        }
+    }
+
+    private static void ensureParentDir(File actualFile) {
         final File parentDir = actualFile.getParentFile();
         if (!parentDir.isDirectory()) {
             if (parentDir.exists()) {
@@ -732,9 +746,6 @@ public class TestExecutor {
                 LOGGER.warn("Unable to create actual file parent dir: " + parentDir);
             }
         }
-        try (FileOutputStream out = new FileOutputStream(actualFile)) {
-            IOUtils.copy(resultStream, out);
-        }
     }
 
     protected HttpResponse executeAndCheckHttpRequest(HttpUriRequest method) throws Exception {
@@ -1624,30 +1635,40 @@ public class TestExecutor {
             resultStream = ResultExtractor.extractStatus(resultStream, UTF_8);
         }
         if (handleVar != null) {
-            String handle = ResultExtractor.extractHandle(resultStream, UTF_8);
+            String result = IOUtils.toString(resultStream, UTF_8);
+            String handle = ResultExtractor.extractHandle(result);
             if (handle != null) {
                 variableCtx.put(handleVar, handle);
+                writeOutputToFile(actualResultFile, result);
+                matchExpectedResult(testFile, expectedResultFile, actualResultFile, resultStream, queryCount,
+                        numResultFiles, compare, statement);
             } else {
                 throw new Exception("no handle for test " + testFile.toString());
             }
         } else if (saveResponseVar != null) {
             variableCtx.put(saveResponseVar, IOUtils.toString(resultStream, UTF_8));
         } else {
-            if (expectedResultFile == null) {
-                if (testFile.getName().startsWith(DIAGNOSE)) {
-                    LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, UTF_8));
-                } else {
-                    LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, UTF_8));
-                    Assert.fail("no result file for " + testFile + "; queryCount: " + queryCount + ", filectxs.size: "
-                            + numResultFiles);
-                }
+            writeOutputToFile(actualResultFile, resultStream);
+            matchExpectedResult(testFile, expectedResultFile, actualResultFile, resultStream, queryCount,
+                    numResultFiles, compare, statement);
+        }
+        queryCount.increment();
+    }
+
+    private void matchExpectedResult(File testFile, File expectedResultFile, File actualResultFile,
+            InputStream resultStream, MutableInt queryCount, int numResultFiles, ComparisonEnum compare,
+            String statement) throws Exception {
+        if (expectedResultFile == null) {
+            if (testFile.getName().startsWith(DIAGNOSE)) {
+                LOGGER.info("Diagnostic output: {}", IOUtils.toString(resultStream, UTF_8));
             } else {
-                writeOutputToFile(actualResultFile, resultStream);
-                runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, UTF_8,
-                        statement);
+                LOGGER.info("Unexpected output: {}", IOUtils.toString(resultStream, UTF_8));
+                Assert.fail("no result file for " + testFile + "; queryCount: " + queryCount + ", filectxs.size: "
+                        + numResultFiles);
             }
+        } else {
+            runScriptAndCompareWithResult(testFile, expectedResultFile, actualResultFile, compare, UTF_8, statement);
         }
-        queryCount.increment();
     }
 
     protected Charset getResponseCharset(File expectedResultFile) {
@@ -2139,7 +2160,7 @@ public class TestExecutor {
             codes.add(Integer.parseInt(m.group(1)));
         }
         if (codes.isEmpty()) {
-            return code -> code == HttpStatus.SC_OK;
+            return code -> code == HttpStatus.SC_OK || code == HttpStatus.SC_ACCEPTED;
         } else {
             return codes::contains;
         }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.async.sqlpp
similarity index 98%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.async.sqlpp
index 66d97f2a81..8d2b1b1a03 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.async.sqlpp
@@ -18,7 +18,6 @@
  */
 
 -- handlevariable=status
-
 select count(*)
 from gargel
 where id = 1;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.1.async.sqlpp
index f8ec2cfa3c..b59b87a326 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.1.async.sqlpp
@@ -19,5 +19,4 @@
 
 -- maxresultreads=1
 -- handlevariable=status
-
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.pollget.http
index 661887f3a9..614e9e062d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.pollget.http
@@ -17,6 +17,6 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+-- polltimeoutsecs=20
+-- handlevariable=result
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.delete.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.delete.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.delete.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.5.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.5.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-exhausted-result/async-exhausted-result.5.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.1.async.sqlpp
index d31a17a2ad..3277705352 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.1.async.sqlpp
@@ -18,6 +18,5 @@
  */
 
 -- handlevariable=status
-
 set `import-private-functions` `true`;
 select value inject_failure(sleep("result", 5000), true);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.2.pollget.http
index 661887f3a9..1fbc9321ef 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-failed/async-failed.2.pollget.http
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=30
+-- handlevariable=result
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.1.async.sqlpp
index e24253ab79..97e3367ecd 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.1.async.sqlpp
@@ -18,5 +18,4 @@
  */
 
 -- handlevariable=status
-
 select i, i * i as i2 from range(1, 5) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.2.pollget.http
index 661887f3a9..d10aed92c1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.2.pollget.http
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=10
+-- handlevariable=result
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.3.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.3.get.http
index 661887f3a9..6496a4b61b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-json/async-json.3.get.http
@@ -16,7 +16,5 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
--- handlevariable=status
-
-select value sleep("result", 5000);
+-- extractresult=true
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.1.async.sqlpp
index 8055915ec8..60e88f4777 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.1.async.sqlpp
@@ -19,5 +19,4 @@
 
 -- maxresultreads=2
 -- handlevariable=status
-
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.2.pollget.http
index 661887f3a9..bca879b735 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.2.pollget.http
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=20
+-- handlevariable=result
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.3.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.3.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.3.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.4.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.4.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.4.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.5.query.sqlpp
similarity index 97%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.5.query.sqlpp
index 1e18f66f59..e45267831b 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-repeated/async-repeated.5.query.sqlpp
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.1.async.sqlpp
index 661887f3a9..3fed1e4e6d 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.1.async.sqlpp
@@ -18,5 +18,4 @@
  */
 
 -- handlevariable=status
-
 select value sleep("result", 5000);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.2.pollget.http
index 661887f3a9..4f793ef19a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.2.pollget.http
@@ -17,6 +17,6 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=10
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.3.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.3.pollget.http
index 661887f3a9..bca879b735 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.3.pollget.http
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=20
+-- handlevariable=result
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.4.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.4.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-running/async-running.4.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.1.async.sqlpp
similarity index 93%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.1.async.sqlpp
index 1e18f66f59..d77d375225 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.1.async.sqlpp
@@ -18,5 +18,5 @@
  */
 
 -- handlevariable=status
-
-select i, i * i as i2 from range(1, 10) i;
+-- param timeout=1s
+select sleep("should not return", 60000);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.2.pollget.http
index 661887f3a9..4f793ef19a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async-timeout/async.2.pollget.http
@@ -17,6 +17,6 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=10
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.1.async.sqlpp
similarity index 99%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.1.async.sqlpp
index 1e18f66f59..5044611755 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.1.async.sqlpp
@@ -18,5 +18,4 @@
  */
 
 -- handlevariable=status
-
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.2.pollget.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.2.pollget.http
index 661887f3a9..d10aed92c1 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.2.pollget.http
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- polltimeoutsecs=10
+-- handlevariable=result
 
-select value sleep("result", 5000);
+$status
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.3.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.3.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/async/async.3.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.1.deferred.sqlpp
similarity index 94%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.1.deferred.sqlpp
index 1e18f66f59..62679638ce 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.1.deferred.sqlpp
@@ -17,6 +17,7 @@
  * under the License.
  */
 
--- handlevariable=status
+-- handlevariable=result
+-- param include-host=false
 
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.2.get.http
similarity index 92%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.2.get.http
index 661887f3a9..b613531d5f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred-improved/deferred/deferred.2.get.http
@@ -17,6 +17,4 @@
  * under the License.
  */
 
--- handlevariable=status
-
-select value sleep("result", 5000);
+$result
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
index a6aef7cb1c..f741692160 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/AsyncDeferredQueries.xml
@@ -69,4 +69,56 @@
             <output-dir compare="Clean-JSON">async-json</output-dir>
         </compilation-unit>
     </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-failed">
+            <output-dir compare="Clean-JSON">async-failed</output-dir>
+            <parameter name="profile" value="timings" type="string"/>
+            <expected-error>Injected failure in inject-failure</expected-error>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-compilation-failed">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">async-compilation-failed</output-dir>
+            <expected-error>Cannot find dataset gargel</expected-error>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="deferred">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">deferred</output-dir>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">async</output-dir>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-repeated">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">async-repeated</output-dir>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-running">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">async-running</output-dir>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-exhausted-result">
+            <output-dir compare="Clean-JSON">async-exhausted-result</output-dir>
+            <parameter name="profile" value="timings" type="string"/>
+            <expected-error>HTTP/1.1 404 Not Found</expected-error>
+            <source-location>false</source-location>
+        </compilation-unit>
+    </test-case>
+    <test-case FilePath="async-deferred-improved">
+        <compilation-unit name="async-json">
+            <parameter name="profile" value="timings" type="string"/>
+            <output-dir compare="Clean-JSON">async-json</output-dir>
+        </compilation-unit>
+    </test-case>
 </test-group>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp
index 66d97f2a81..e63fd76058 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-compilation-failed/async-compilation-failed.1.async.sqlpp
@@ -18,6 +18,7 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 select count(*)
 from gargel
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp
index f8ec2cfa3c..48872a185e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-exhausted-result/async-exhausted-result.1.async.sqlpp
@@ -19,5 +19,6 @@
 
 -- maxresultreads=1
 -- handlevariable=status
+-- param include-host=true
 
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp
index d31a17a2ad..b95cca0115 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-failed/async-failed.1.async.sqlpp
@@ -18,6 +18,7 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 set `import-private-functions` `true`;
 select value inject_failure(sleep("result", 5000), true);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp
index e24253ab79..950fc1ae6e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-json/async-json.1.async.sqlpp
@@ -18,5 +18,6 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 select i, i * i as i2 from range(1, 5) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp
index 8055915ec8..532844560a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-repeated/async-repeated.1.async.sqlpp
@@ -19,5 +19,6 @@
 
 -- maxresultreads=2
 -- handlevariable=status
+-- param include-host=true
 
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
index 661887f3a9..80f4868da0 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async-running/async-running.1.async.sqlpp
@@ -18,5 +18,6 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 select value sleep("result", 5000);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
index 1e18f66f59..56b8960d5c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/async-deferred/async/async.1.async.sqlpp
@@ -18,5 +18,6 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 select i, i * i as i2 from range(1, 10) i;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/flwor/at00/at00.6.async.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/flwor/at00/at00.6.async.sqlpp
index 10b8c85816..cdf70f3fe2 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/flwor/at00/at00.6.async.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/flwor/at00/at00.6.async.sqlpp
@@ -23,6 +23,7 @@
  */
 
 -- handlevariable=status
+-- param include-host=true
 
 use test;
 
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.regex
new file mode 100644
index 0000000000..66de2a0a08
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-compilation-failed/async-compilation-failed.1.regex
@@ -0,0 +1,2 @@
+/"status": "failed"/
+/"errors": ".*"/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.regexjson
new file mode 100644
index 0000000000..0ed7a67c58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.2.regexjson
@@ -0,0 +1,8 @@
+{
+  "status":"success",
+  "handle": "R{.*}",
+  "resultCount": 10,
+  "partitions": "R{.*}",
+  "metrics": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.regexjson
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.3.regexjson
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.ignore
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-exhausted-result/async-exhausted-result.4.ignore
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-failed/async-failed.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-failed/async-failed.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-failed/async-failed.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-failed/async-failed.2.regex
new file mode 100644
index 0000000000..66de2a0a08
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-failed/async-failed.2.regex
@@ -0,0 +1,2 @@
+/"status": "failed"/
+/"errors": ".*"/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.2.regexjson
new file mode 100644
index 0000000000..c6c829ba60
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.2.regexjson
@@ -0,0 +1,8 @@
+{
+  "status":"success",
+  "handle": "R{.*}",
+  "resultCount": 5,
+  "partitions": "R{.*}",
+  "metrics": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.3.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.3.json
new file mode 100644
index 0000000000..2b249bcd1e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-json/async-json.3.json
@@ -0,0 +1,5 @@
+{ "i": 1, "i2": 1 }
+{ "i": 2, "i2": 4 }
+{ "i": 3, "i2": 9 }
+{ "i": 4, "i2": 16 }
+{ "i": 5, "i2": 25 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.2.regexjson
new file mode 100644
index 0000000000..0ed7a67c58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.2.regexjson
@@ -0,0 +1,8 @@
+{
+  "status":"success",
+  "handle": "R{.*}",
+  "resultCount": 10,
+  "partitions": "R{.*}",
+  "metrics": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.3.regexjson
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.3.regexjson
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.4.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.4.regexjson
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.4.regexjson
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.5.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.5.json
new file mode 100644
index 0000000000..09e86ccf89
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-repeated/async-repeated.5.json
@@ -0,0 +1,10 @@
+{ "i": 1, "i2": 1 }
+{ "i": 2, "i2": 4 }
+{ "i": 3, "i2": 9 }
+{ "i": 4, "i2": 16 }
+{ "i": 5, "i2": 25 }
+{ "i": 6, "i2": 36 }
+{ "i": 7, "i2": 49 }
+{ "i": 8, "i2": 64 }
+{ "i": 9, "i2": 81 }
+{ "i": 10, "i2": 100 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.2.json b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.2.json
new file mode 100644
index 0000000000..272762e81c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.2.json
@@ -0,0 +1,3 @@
+{
+	"status": "running"
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.3.regexjson
new file mode 100644
index 0000000000..9e724fd104
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.3.regexjson
@@ -0,0 +1,8 @@
+{
+  "status":"success",
+  "handle": "R{.*}",
+  "resultCount": 1,
+  "partitions": "R{.*}",
+  "metrics": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.4.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.4.regexjson
new file mode 100644
index 0000000000..9bc04e0c28
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-running/async-running.4.regexjson
@@ -0,0 +1,8 @@
+{
+	"results": [
+	  "result"
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.2.regex
new file mode 100644
index 0000000000..b1fee72721
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async-timeout/async-timeout.2.regex
@@ -0,0 +1,3 @@
+/"status": "timeout"/
+/"errors":.*/
+/.*"msg": "ASX0030: Request timed out and will be cancelled".*/
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.2.regexjson
new file mode 100644
index 0000000000..0ed7a67c58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.2.regexjson
@@ -0,0 +1,8 @@
+{
+  "status":"success",
+  "handle": "R{.*}",
+  "resultCount": 10,
+  "partitions": "R{.*}",
+  "metrics": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.3.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.3.regexjson
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/async/async.3.regexjson
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.1.ignore b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.1.ignore
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.2.regexjson b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.2.regexjson
new file mode 100644
index 0000000000..72718f7933
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/async-deferred-improved/deferred/deferred.2.regexjson
@@ -0,0 +1,17 @@
+{
+	"results": [
+	  { "i": 1, "i2": 1 },
+	  { "i": 2, "i2": 4 },
+	  { "i": 3, "i2": 9 },
+	  { "i": 4, "i2": 16 },
+	  { "i": 5, "i2": 25 },
+	  { "i": 6, "i2": 36 },
+	  { "i": 7, "i2": 49 },
+	  { "i": 8, "i2": 64 },
+	  { "i": 9, "i2": 81 },
+	  { "i": 10, "i2": 100 }
+  ],
+  "metrics": "R{.*}",
+  "profile": "R{.*}",
+  "createdAt": "R{.*}"
+}
\ No newline at end of file
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestTracker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestTracker.java
index 230eac3f50..dbae70e144 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestTracker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IRequestTracker.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.common.api;
 
 import java.util.Collection;
+import java.util.Optional;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.job.IJobLifecycleListener;
@@ -92,4 +93,11 @@ public interface IRequestTracker extends IJobLifecycleListener {
      * @return the total number of failed requests
      */
     long getTotalNumberOfFailedRequests();
+
+    void trackAsyncOrDeferredRequest(IClientRequest request);
+
+    void removeAsyncOrDeferredRequest(String requestId);
+
+    Optional<IClientRequest> getAsyncOrDeferredRequest(String requestId);
+
 }
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RequestTracker.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RequestTracker.java
index ef9154d887..24bbd0611c 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RequestTracker.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/RequestTracker.java
@@ -25,6 +25,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.AtomicLong;
 
@@ -43,6 +44,7 @@ public class RequestTracker implements IRequestTracker {
     private final Map<String, IClientRequest> runningRequests = new ConcurrentHashMap<>();
     private final Map<String, IClientRequest> clientIdRequests = new ConcurrentHashMap<>();
     private final Map<String, IClientRequest> completedRequests;
+    private final Map<String, IClientRequest> asyncRequests = new ConcurrentHashMap<>();
     private final ICcApplicationContext ccAppCtx;
     private final AtomicLong numRequests;
     private final AtomicLong numOfFailedRequests;
@@ -82,6 +84,22 @@ public class RequestTracker implements IRequestTracker {
         }
     }
 
+    @Override
+    public void trackAsyncOrDeferredRequest(IClientRequest request) {
+        asyncRequests.put(request.getId(), request);
+    }
+
+    @Override
+    public void removeAsyncOrDeferredRequest(String requestId) {
+        if (requestId != null) {
+            asyncRequests.remove(requestId);
+        }
+    }
+
+    public Optional<IClientRequest> getAsyncOrDeferredRequest(String requestId) {
+        return Optional.ofNullable(requestId).map(asyncRequests::get);
+    }
+
     @Override
     public void cancel(String requestId) throws HyracksDataException {
         final IClientRequest request = runningRequests.get(requestId);
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
index f107fb802f..65ae2ab511 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultPartitionManager.java
@@ -30,7 +30,8 @@ public interface IResultPartitionManager extends IResultManager {
     void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, int partition, int nPartitions,
             IResultMetadata metadata, boolean emptyResult) throws HyracksException;
 
-    void reportPartitionWriteCompletion(JobId jobId, ResultSetId resultSetId, int partition) throws HyracksException;
+    void reportPartitionWriteCompletion(JobId jobId, ResultSetId resultSetId, int partition, int resultCount)
+            throws HyracksException;
 
     void reportPartitionConsumed(JobId jobId, ResultSetId rsId, int partition) throws HyracksException;
 
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
index ff2e48a349..20b43bae78 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSet.java
@@ -23,4 +23,7 @@ import org.apache.hyracks.api.job.JobId;
 
 public interface IResultSet {
     IResultSetReader createReader(JobId jobId, ResultSetId resultSetId) throws HyracksDataException;
+
+    IResultSetReader createReader(JobId jobId, ResultSetId resultSetId, int partition) throws HyracksDataException;
+
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
index 1652dd42f4..b06f5d71e3 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/IResultSetReader.java
@@ -33,4 +33,6 @@ public interface IResultSetReader {
      * @return the result metadata
      */
     IResultMetadata getResultMetadata();
+
+    ResultDirectoryRecord[] getResultRecords() throws HyracksDataException;
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
index a442c11c85..b56d36c4e4 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultDirectoryRecord.java
@@ -30,7 +30,7 @@ public class ResultDirectoryRecord implements Serializable {
         FAILED
     }
 
-    private static final long serialVersionUID = 1L;
+    private static final long serialVersionUID = 2L;
 
     private NetworkAddress address;
 
@@ -40,10 +40,15 @@ public class ResultDirectoryRecord implements Serializable {
 
     private boolean empty;
 
+    private int resultCount;
+
+    private String nodeId;
+
     public ResultDirectoryRecord() {
         this.address = null;
         this.readEOS = false;
         this.status = Status.IDLE;
+        this.nodeId = null;
     }
 
     public void setNetworkAddress(NetworkAddress address) {
@@ -54,10 +59,22 @@ public class ResultDirectoryRecord implements Serializable {
         return address;
     }
 
+    public void setNodeId(String nodeId) {
+        this.nodeId = nodeId;
+    }
+
+    public String getNodeId() {
+        return nodeId;
+    }
+
     public void setEmpty(boolean empty) {
         this.empty = empty;
     }
 
+    public void setResultCount(int resultCount) {
+        this.resultCount = resultCount;
+    }
+
     public boolean isEmpty() {
         return empty;
     }
@@ -86,6 +103,10 @@ public class ResultDirectoryRecord implements Serializable {
         status = Status.FAILED;
     }
 
+    public int getResultCount() {
+        return resultCount;
+    }
+
     private void updateStatus(final ResultDirectoryRecord.Status newStatus) {
         // FAILED is a stable status
         if (status != Status.FAILED) {
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
index 44de45f771..edda92fedb 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/result/ResultJobRecord.java
@@ -35,7 +35,8 @@ public class ResultJobRecord implements IResultStateRecord {
         RUNNING,
         SUCCESS,
         REMOVED,
-        FAILED
+        FAILED,
+        TIMEOUT
     }
 
     public static class Status implements Serializable {
@@ -85,15 +86,17 @@ public class ResultJobRecord implements IResultStateRecord {
     private Status status;
     private ResultSetId rsId;
     private ResultSetMetaData resultSetMetaData;
+    private long resultCount;
 
     public ResultJobRecord() {
         this.timestamp = System.nanoTime();
         this.status = new Status();
+        this.resultCount = 0;
     }
 
     private void updateState(State newStatus) {
         // FAILED is a stable status
-        if (status.state != State.FAILED) {
+        if (status.state != State.FAILED && status.state != State.TIMEOUT) {
             status.setState(newStatus);
         }
     }
@@ -111,8 +114,13 @@ public class ResultJobRecord implements IResultStateRecord {
         return jobEndTime - jobStartTime;
     }
 
-    public void success() {
+    public long getResultCount() {
+        return resultCount;
+    }
+
+    public void success(int resultCount) {
         updateState(State.SUCCESS);
+        this.resultCount = resultCount;
     }
 
     public void consume() {
@@ -128,6 +136,10 @@ public class ResultJobRecord implements IResultStateRecord {
         status.setExceptions(exceptions);
     }
 
+    public void timeout() {
+        updateState(State.TIMEOUT);
+    }
+
     @Override
     public long getTimestamp() {
         return timestamp;
@@ -176,11 +188,13 @@ public class ResultJobRecord implements IResultStateRecord {
     public synchronized void updateState() {
         int successCount = 0;
         int consumedCount = 0;
+        int resultCount = 0;
         ResultDirectoryRecord[] records = resultSetMetaData.getRecords();
         for (ResultDirectoryRecord record : records) {
             if (record != null) {
                 if (record.ready()) {
                     successCount++;
+                    resultCount += record.getResultCount();
                 }
                 if (record.hasReachedReadEOS()) {
                     consumedCount++;
@@ -190,7 +204,7 @@ public class ResultJobRecord implements IResultStateRecord {
         if (consumedCount == records.length) {
             consume();
         } else if (successCount == records.length) {
-            success();
+            success(resultCount);
         }
     }
 
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java
new file mode 100644
index 0000000000..dd72e2e81d
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/PartitionResultSetReader.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.client.result;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hyracks.api.comm.FrameHelper;
+import org.apache.hyracks.api.comm.IFrame;
+import org.apache.hyracks.api.context.IHyracksCommonContext;
+import org.apache.hyracks.api.exceptions.ErrorCode;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.result.IResultDirectory;
+import org.apache.hyracks.api.result.ResultDirectoryRecord;
+import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.client.net.ClientNetworkManager;
+
+public class PartitionResultSetReader extends ResultSetReader {
+
+    private boolean firstRead;
+
+    public PartitionResultSetReader(IResultDirectory resultDirectory, ClientNetworkManager netManager,
+            IHyracksCommonContext resultClientCtx, JobId jobId, ResultSetId resultSetId, int partitionId) {
+        super(resultDirectory, netManager, resultClientCtx, jobId, resultSetId);
+        this.currentRecord = partitionId;
+        this.firstRead = true;
+    }
+
+    @Override
+    public int read(IFrame frame) throws HyracksDataException {
+        frame.reset();
+        int readSize = 0;
+        try {
+            if (isFirstRead() && !hasNextRecord()) {
+                return readSize;
+            }
+            // read until frame is full or all result records have been read
+            while (readSize < frame.getFrameSize()) {
+                if (currentRecordMonitor.hasMoreFrames()) {
+                    final ByteBuffer readBuffer = currentRecordChannel.getNextBuffer();
+                    if (readBuffer == null) {
+                        throw new IllegalStateException("Unexpected empty frame");
+                    }
+                    currentRecordMonitor.notifyFrameRead();
+                    if (readSize == 0) {
+                        final int nBlocks = FrameHelper.deserializeNumOfMinFrame(readBuffer);
+                        frame.ensureFrameSize(frame.getMinSize() * nBlocks);
+                        frame.getBuffer().clear();
+                    }
+                    frame.getBuffer().put(readBuffer);
+                    currentRecordChannel.recycleBuffer(readBuffer);
+                    readSize = frame.getBuffer().position();
+                } else {
+                    currentRecordChannel.close();
+                    if (currentRecordMonitor.failed()) {
+                        throw HyracksDataException.create(ErrorCode.FAILED_TO_READ_RESULT, jobId);
+                    }
+                    break;
+                }
+            }
+            frame.getBuffer().flip();
+        } catch (Exception e) {
+            if (isLocalFailure()) {
+                currentRecordChannel.fail();
+            }
+            throw e;
+        }
+        return readSize;
+    }
+
+    @Override
+    protected boolean hasNextRecord() throws HyracksDataException {
+        firstRead = false;
+        try {
+            knownRecords = resultDirectory.getResultLocations(jobId, resultSetId, knownRecords);
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        if (currentRecord >= knownRecords.length) {
+            // exhausted all known records
+            return false;
+        }
+        ResultDirectoryRecord record = getRecord(currentRecord);
+        // skip empty records
+        if (record.isEmpty()) {
+            return false;
+        }
+        requestRecordData(record);
+        return true;
+    }
+
+    @Override
+    protected boolean isFirstRead() {
+        return firstRead;
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
index 8a88045fc3..1d1ede8f8c 100644
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSet.java
@@ -73,6 +73,19 @@ public class ResultSet implements IResultSet, Closeable {
         return reader;
     }
 
+    @Override
+    public IResultSetReader createReader(JobId jobId, ResultSetId resultSetId, int partition)
+            throws HyracksDataException {
+        IResultSetReader reader = null;
+        try {
+            reader = new PartitionResultSetReader(resultDirectory, netManager, resultClientCtx, jobId, resultSetId,
+                    partition);
+        } catch (Exception e) {
+            throw HyracksDataException.create(e);
+        }
+        return reader;
+    }
+
     static class ResultClientContext extends FrameManager implements IHyracksCommonContext {
 
         ResultClientContext(int frameSize) {
diff --git a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
index 992c8bebf3..9765805e8d 100644
--- a/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
+++ b/hyracks-fullstack/hyracks/hyracks-client/src/main/java/org/apache/hyracks/client/result/ResultSetReader.java
@@ -46,18 +46,18 @@ import org.apache.logging.log4j.Logger;
 @NotThreadSafe
 public class ResultSetReader implements IResultSetReader {
 
-    private static final Logger LOGGER = LogManager.getLogger();
-    private static final int NUM_READ_BUFFERS = 1;
-    private final IResultDirectory resultDirectory;
-    private final ClientNetworkManager netManager;
-    private final IHyracksCommonContext resultClientCtx;
-    private final JobId jobId;
-    private final ResultSetId resultSetId;
-    private ResultDirectoryRecord[] knownRecords;
-    private ResultInputChannelMonitor[] monitors;
-    private ResultInputChannelMonitor currentRecordMonitor;
-    private ResultNetworkInputChannel currentRecordChannel;
-    private int currentRecord;
+    protected static final Logger LOGGER = LogManager.getLogger();
+    protected static final int NUM_READ_BUFFERS = 1;
+    protected final IResultDirectory resultDirectory;
+    protected final ClientNetworkManager netManager;
+    protected final IHyracksCommonContext resultClientCtx;
+    protected final JobId jobId;
+    protected final ResultSetId resultSetId;
+    protected ResultDirectoryRecord[] knownRecords;
+    protected ResultInputChannelMonitor[] monitors;
+    protected ResultInputChannelMonitor currentRecordMonitor;
+    protected ResultNetworkInputChannel currentRecordChannel;
+    protected int currentRecord;
 
     public ResultSetReader(IResultDirectory resultDirectory, ClientNetworkManager netManager,
             IHyracksCommonContext resultClientCtx, JobId jobId, ResultSetId resultSetId) {
@@ -162,7 +162,7 @@ public class ResultSetReader implements IResultSetReader {
         return monitors[partition];
     }
 
-    private boolean hasNextRecord() throws HyracksDataException {
+    protected boolean hasNextRecord() throws HyracksDataException {
         currentRecord++;
         ResultDirectoryRecord record = getRecord(currentRecord);
         // skip empty records
@@ -177,7 +177,7 @@ public class ResultSetReader implements IResultSetReader {
         return true;
     }
 
-    private ResultDirectoryRecord getRecord(int recordNum) throws HyracksDataException {
+    protected ResultDirectoryRecord getRecord(int recordNum) throws HyracksDataException {
         try {
             while (knownRecords == null || knownRecords[recordNum] == null) {
                 knownRecords = resultDirectory.getResultLocations(jobId, resultSetId, knownRecords);
@@ -188,7 +188,18 @@ public class ResultSetReader implements IResultSetReader {
         }
     }
 
-    private void requestRecordData(ResultDirectoryRecord record) throws HyracksDataException {
+    public ResultDirectoryRecord[] getResultRecords() throws HyracksDataException {
+        while (knownRecords == null) {
+            try {
+                knownRecords = resultDirectory.getResultLocations(jobId, resultSetId, knownRecords);
+            } catch (Exception e) {
+                throw HyracksDataException.create(e);
+            }
+        }
+        return knownRecords;
+    }
+
+    protected void requestRecordData(ResultDirectoryRecord record) throws HyracksDataException {
         currentRecordChannel = new ResultNetworkInputChannel(netManager, getSocketAddress(record), jobId, resultSetId,
                 currentRecord, NUM_READ_BUFFERS);
         currentRecordMonitor = getMonitor(currentRecord);
@@ -196,19 +207,19 @@ public class ResultSetReader implements IResultSetReader {
         currentRecordChannel.open(resultClientCtx);
     }
 
-    private boolean isFirstRead() {
+    protected boolean isFirstRead() {
         return currentRecord == -1;
     }
 
-    private boolean isLastRecord() {
+    protected boolean isLastRecord() {
         return knownRecords != null && currentRecord == knownRecords.length - 1;
     }
 
-    private boolean isLocalFailure() {
+    protected boolean isLocalFailure() {
         return currentRecordMonitor != null && !currentRecordMonitor.failed();
     }
 
-    private static class ResultInputChannelMonitor implements IInputChannelMonitor {
+    protected static class ResultInputChannelMonitor implements IInputChannelMonitor {
 
         private int availableFrames;
         private boolean eos;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
index 6d32621a4a..a95b33bef3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/ClusterControllerIPCI.java
@@ -120,13 +120,13 @@ class ClusterControllerIPCI implements IIPCI {
                 ccs.getWorkQueue()
                         .schedule(new RegisterResultPartitionLocationWork(ccs, rrplf.getJobId(), rrplf.getResultSetId(),
                                 rrplf.getMetadata(), rrplf.getEmptyResult(), rrplf.getPartition(),
-                                rrplf.getNPartitions(), rrplf.getNetworkAddress()));
+                                rrplf.getNPartitions(), rrplf.getNetworkAddress(), rrplf.getNodeId()));
                 break;
             case REPORT_RESULT_PARTITION_WRITE_COMPLETION:
                 CCNCFunctions.ReportResultPartitionWriteCompletionFunction rrpwc =
                         (CCNCFunctions.ReportResultPartitionWriteCompletionFunction) fn;
                 ccs.getWorkQueue().schedule(new ReportResultPartitionWriteCompletionWork(ccs, rrpwc.getJobId(),
-                        rrpwc.getResultSetId(), rrpwc.getPartition()));
+                        rrpwc.getResultSetId(), rrpwc.getPartition(), rrpwc.getResultCount()));
                 break;
             case REPORT_RESULT_PARTITION_CONSUMED:
                 CCNCFunctions.ReportResultPartitionConsumedFunction rrpc =
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
index a986f331ee..69a420a25c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/IResultDirectoryService.java
@@ -37,16 +37,18 @@ public interface IResultDirectoryService extends IJobLifecycleListener, IResultM
     public void init(ExecutorService executor, IJobResultCallback callback);
 
     public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, IResultMetadata metadata,
-            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
+            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress, String nodeId)
             throws HyracksDataException;
 
-    public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
+    public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition, int resultCount)
             throws HyracksDataException;
 
     public void reportResultPartitionConsumed(JobId jobId, ResultSetId rsId, int partition) throws HyracksDataException;
 
     public void reportJobFailure(JobId jobId, List<Exception> exceptions);
 
+    public void reportJobTimeout(JobId jobId);
+
     public Status getResultStatus(JobId jobId, ResultSetId rsId) throws HyracksDataException;
 
     public IResultMetadata getResultMetadata(JobId jobId, ResultSetId rsId) throws HyracksDataException;
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
index 139775c353..6a1d28fc4d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/result/ResultDirectoryService.java
@@ -101,6 +101,8 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
             }
             resultJobRecord.finish();
             jobResultCallback.completed(jobId, resultJobRecord);
+        } else {
+            reportJobFailure(jobId, exceptions);
         }
     }
 
@@ -119,7 +121,7 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
 
     @Override
     public synchronized void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, IResultMetadata metadata,
-            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress)
+            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress, String nodeId)
             throws HyracksDataException {
         ResultJobRecord djr = getNonNullResultJobRecord(jobId);
         djr.setResultSetMetaData(rsId, metadata, nPartitions);
@@ -127,6 +129,7 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
 
         record.setNetworkAddress(networkAddress);
         record.setEmpty(emptyResult);
+        record.setNodeId(nodeId);
         record.start();
 
         final JobResultInfo jobResultInfo = jobResultLocations.get(jobId);
@@ -146,10 +149,11 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
     }
 
     @Override
-    public synchronized void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition)
-            throws HyracksDataException {
+    public synchronized void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition,
+            int resultCount) throws HyracksDataException {
         ResultJobRecord djr = getNonNullResultJobRecord(jobId);
         djr.getDirectoryRecord(partition).writeEOS();
+        djr.getDirectoryRecord(partition).setResultCount(resultCount);
         djr.updateState();
         notifyAll();
     }
@@ -183,6 +187,14 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
         notifyAll();
     }
 
+    @Override
+    public synchronized void reportJobTimeout(JobId jobId) {
+        ResultJobRecord rjr = getResultJobRecord(jobId);
+        if (rjr != null) {
+            rjr.timeout();
+        }
+    }
+
     @Override
     public synchronized ResultJobRecord.Status getResultStatus(JobId jobId, ResultSetId rsId)
             throws HyracksDataException {
@@ -191,7 +203,11 @@ public class ResultDirectoryService extends AbstractResultManager implements IRe
 
     @Override
     public synchronized IResultMetadata getResultMetadata(JobId jobId, ResultSetId rsId) throws HyracksDataException {
-        return getNonNullResultJobRecord(jobId).getResultSetMetaData().getMetadata();
+        ResultSetMetaData resultSetMetaData = getNonNullResultJobRecord(jobId).getResultSetMetaData();
+        if (resultSetMetaData == null) {
+            throw HyracksDataException.create(ErrorCode.NO_RESULT_SET, jobId);
+        }
+        return resultSetMetaData.getMetadata();
     }
 
     @Override
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultMetadataWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultMetadataWork.java
index f2ad7d350e..155f08123b 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultMetadataWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetResultMetadataWork.java
@@ -28,11 +28,8 @@ import org.apache.hyracks.control.common.work.SynchronizableWork;
 
 public class GetResultMetadataWork extends SynchronizableWork {
     private final ClusterControllerService ccs;
-
     private final JobId jobId;
-
     private final ResultSetId rsId;
-
     private final IResultCallback<IResultMetadata> callback;
 
     public GetResultMetadataWork(ClusterControllerService ccs, JobId jobId, ResultSetId rsId,
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
index d51776110b..579c6bdfc3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/RegisterResultPartitionLocationWork.java
@@ -54,9 +54,11 @@ public class RegisterResultPartitionLocationWork extends AbstractWork {
 
     private final NetworkAddress networkAddress;
 
+    private final String nodeId;
+
     public RegisterResultPartitionLocationWork(ClusterControllerService ccs, JobId jobId, ResultSetId rsId,
             IResultMetadata metadata, boolean emptyResult, int partition, int nPartitions,
-            NetworkAddress networkAddress) {
+            NetworkAddress networkAddress, String nodeId) {
         this.ccs = ccs;
         this.jobId = jobId;
         this.rsId = rsId;
@@ -65,13 +67,14 @@ public class RegisterResultPartitionLocationWork extends AbstractWork {
         this.partition = partition;
         this.nPartitions = nPartitions;
         this.networkAddress = networkAddress;
+        this.nodeId = nodeId;
     }
 
     @Override
     public void run() {
         try {
             ccs.getResultDirectoryService().registerResultPartitionLocation(jobId, rsId, metadata, emptyResult,
-                    partition, nPartitions, networkAddress);
+                    partition, nPartitions, networkAddress, nodeId);
         } catch (HyracksDataException e) {
             LOGGER.log(Level.WARN, "Failed to register partition location", e);
             // Should fail the job if exists on cc, otherwise, do nothing
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
index a07c31c769..bc161f8d20 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/ReportResultPartitionWriteCompletionWork.java
@@ -33,18 +33,21 @@ public class ReportResultPartitionWriteCompletionWork extends AbstractWork {
 
     private final int partition;
 
+    private final int resultCount;
+
     public ReportResultPartitionWriteCompletionWork(ClusterControllerService ccs, JobId jobId, ResultSetId rsId,
-            int partition) {
+            int partition, int resultCount) {
         this.ccs = ccs;
         this.jobId = jobId;
         this.rsId = rsId;
         this.partition = partition;
+        this.resultCount = resultCount;
     }
 
     @Override
     public void run() {
         try {
-            ccs.getResultDirectoryService().reportResultPartitionWriteCompletion(jobId, rsId, partition);
+            ccs.getResultDirectoryService().reportResultPartitionWriteCompletion(jobId, rsId, partition, resultCount);
         } catch (HyracksDataException e) {
             throw new RuntimeException(e);
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
index d706c58ef8..738185712c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/base/IClusterController.java
@@ -69,9 +69,10 @@ public interface IClusterController {
     void sendRealTimeApplicationMessageToCC(byte[] data, DeploymentId deploymentId, String nodeId) throws Exception;
 
     void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, IResultMetadata metadata, boolean emptyResult,
-            int partition, int nPartitions, NetworkAddress networkAddress) throws Exception;
+            int partition, int nPartitions, NetworkAddress networkAddress, String nodeId) throws Exception;
 
-    void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition) throws Exception;
+    void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition, int resultCount)
+            throws Exception;
 
     void reportResultPartitionConsumed(JobId jobId, ResultSetId rsId, int partition) throws Exception;
 
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
index ace60955d4..333f07cb00 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/CCNCFunctions.java
@@ -609,8 +609,10 @@ public class CCNCFunctions {
 
         private NetworkAddress networkAddress;
 
+        private String nodeId;
+
         public RegisterResultPartitionLocationFunction(JobId jobId, ResultSetId rsId, IResultMetadata metadata,
-                boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress) {
+                boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress, String nodeId) {
             this.jobId = jobId;
             this.rsId = rsId;
             this.metadata = metadata;
@@ -618,6 +620,7 @@ public class CCNCFunctions {
             this.partition = partition;
             this.nPartitions = nPartitions;
             this.networkAddress = networkAddress;
+            this.nodeId = nodeId;
         }
 
         @Override
@@ -652,6 +655,10 @@ public class CCNCFunctions {
         public NetworkAddress getNetworkAddress() {
             return networkAddress;
         }
+
+        public String getNodeId() {
+            return nodeId;
+        }
     }
 
     public static class ReportResultPartitionWriteCompletionFunction extends Function {
@@ -663,10 +670,14 @@ public class CCNCFunctions {
 
         private final int partition;
 
-        public ReportResultPartitionWriteCompletionFunction(JobId jobId, ResultSetId rsId, int partition) {
+        private final int resultCount;
+
+        public ReportResultPartitionWriteCompletionFunction(JobId jobId, ResultSetId rsId, int partition,
+                int resultCount) {
             this.jobId = jobId;
             this.rsId = rsId;
             this.partition = partition;
+            this.resultCount = resultCount;
         }
 
         @Override
@@ -685,6 +696,10 @@ public class CCNCFunctions {
         public int getPartition() {
             return partition;
         }
+
+        public int getResultCount() {
+            return resultCount;
+        }
     }
 
     public static class ReportResultPartitionConsumedFunction extends Function {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
index 5313dad496..46b192484e 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
@@ -142,16 +142,18 @@ public class ClusterControllerRemoteProxy implements IClusterController {
 
     @Override
     public void registerResultPartitionLocation(JobId jobId, ResultSetId rsId, IResultMetadata metadata,
-            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress) throws Exception {
+            boolean emptyResult, int partition, int nPartitions, NetworkAddress networkAddress, String nodeId)
+            throws Exception {
         RegisterResultPartitionLocationFunction fn = new RegisterResultPartitionLocationFunction(jobId, rsId, metadata,
-                emptyResult, partition, nPartitions, networkAddress);
+                emptyResult, partition, nPartitions, networkAddress, nodeId);
         ipcHandle.send(-1, fn, null);
     }
 
     @Override
-    public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition) throws Exception {
+    public void reportResultPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition, int resultCount)
+            throws Exception {
         ReportResultPartitionWriteCompletionFunction fn =
-                new ReportResultPartitionWriteCompletionFunction(jobId, rsId, partition);
+                new ReportResultPartitionWriteCompletionFunction(jobId, rsId, partition, resultCount);
         ipcHandle.send(-1, fn, null);
     }
 
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
index 557b74ba22..877db8a83a 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionManager.java
@@ -92,18 +92,21 @@ public class ResultPartitionManager extends AbstractResultManager implements IRe
         try {
             // Be sure to send the *public* network address to the CC
             ncs.getClusterController(jobId.getCcId()).registerResultPartitionLocation(jobId, rsId, metadata,
-                    emptyResult, partition, nPartitions, ncs.getResultNetworkManager().getPublicNetworkAddress());
+                    emptyResult, partition, nPartitions, ncs.getResultNetworkManager().getPublicNetworkAddress(),
+                    ncs.getId());
         } catch (Exception e) {
             throw HyracksException.create(e);
         }
     }
 
     @Override
-    public void reportPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition) throws HyracksException {
+    public void reportPartitionWriteCompletion(JobId jobId, ResultSetId rsId, int partition, int resultCount)
+            throws HyracksException {
         try {
             LOGGER.trace("Reporting partition write completion: JobId: {}:ResultSetId: {}:partition: {}", jobId, rsId,
                     partition);
-            ncs.getClusterController(jobId.getCcId()).reportResultPartitionWriteCompletion(jobId, rsId, partition);
+            ncs.getClusterController(jobId.getCcId()).reportResultPartitionWriteCompletion(jobId, rsId, partition,
+                    resultCount);
         } catch (Exception e) {
             throw HyracksException.create(e);
         }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
index 55db15039e..aec569704c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultPartitionWriter.java
@@ -58,6 +58,8 @@ public class ResultPartitionWriter implements IFrameWriter {
 
     private boolean failed = false;
 
+    private int resultCount;
+
     public ResultPartitionWriter(IHyracksTaskContext ctx, IResultPartitionManager manager, JobId jobId,
             ResultSetId rsId, boolean asyncMode, IResultMetadata metadata, int partition, int nPartitions,
             ResultMemoryManager resultMemoryManager, IWorkspaceFileFactory fileFactory, long maxReads) {
@@ -72,6 +74,7 @@ public class ResultPartitionWriter implements IFrameWriter {
         resultSetPartitionId = new ResultSetPartitionId(jobId, rsId, partition);
         resultState = new ResultState(resultSetPartitionId, asyncMode, ctx.getIoManager(), fileFactory,
                 ctx.getInitialFrameSize(), maxReads);
+        resultCount = 0;
     }
 
     public ResultState getResultState() {
@@ -118,7 +121,7 @@ public class ResultPartitionWriter implements IFrameWriter {
         }
         try {
             if (partitionRegistered) {
-                manager.reportPartitionWriteCompletion(jobId, resultSetId, partition);
+                manager.reportPartitionWriteCompletion(jobId, resultSetId, partition, resultCount);
             }
         } catch (HyracksException e) {
             throw HyracksDataException.create(e);
@@ -135,4 +138,8 @@ public class ResultPartitionWriter implements IFrameWriter {
             throw HyracksDataException.create(e);
         }
     }
+
+    public void incrementResultCount() {
+        resultCount++;
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
index 25d3f00364..880afb7b60 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/result/ResultState.java
@@ -42,6 +42,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 
 public class ResultState implements IStateObject {
     private static final String FILE_PREFIX = "result_";
+    public static final long UNLIMITED_READS = -1;
 
     private final ResultSetPartitionId resultSetPartitionId;
 
@@ -69,12 +70,14 @@ public class ResultState implements IStateObject {
 
     private long persistentSize;
     private long remainingReads;
+    private long maxReads;
 
     ResultState(ResultSetPartitionId resultSetPartitionId, boolean asyncMode, IIOManager ioManager,
             IWorkspaceFileFactory fileFactory, int frameSize, long maxReads) {
-        if (maxReads <= 0) {
-            throw new IllegalArgumentException("maxReads must be > 0");
+        if (maxReads < -1) {
+            throw new IllegalArgumentException("maxReads must be >= -1");
         }
+        this.maxReads = maxReads;
         this.resultSetPartitionId = resultSetPartitionId;
         this.asyncMode = asyncMode;
         this.ioManager = ioManager;
@@ -376,6 +379,6 @@ public class ResultState implements IStateObject {
     }
 
     public synchronized boolean isExhausted() {
-        return remainingReads == 0;
+        return maxReads != UNLIMITED_READS && remainingReads == 0;
     }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
index 9229af65f5..29b37c8922 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/pom.xml
@@ -83,7 +83,6 @@
       <groupId>org.apache.hyracks</groupId>
       <artifactId>hyracks-control-nc</artifactId>
       <version>${project.version}</version>
-      <scope>test</scope>
     </dependency>
     <dependency>
       <groupId>org.apache.hyracks</groupId>
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
index 219b22cc5f..a8116662ea 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/result/ResultWriterOperatorDescriptor.java
@@ -38,6 +38,7 @@ import org.apache.hyracks.api.job.IOperatorDescriptorRegistry;
 import org.apache.hyracks.api.result.IResultMetadata;
 import org.apache.hyracks.api.result.IResultPartitionManager;
 import org.apache.hyracks.api.result.ResultSetId;
+import org.apache.hyracks.control.nc.result.ResultPartitionWriter;
 import org.apache.hyracks.dataflow.common.comm.io.FrameOutputStream;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
 import org.apache.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -109,6 +110,7 @@ public class ResultWriterOperatorDescriptor extends AbstractSingleActivityOperat
                                     frameOutputStream.getLength());
                         }
                     }
+                    ((ResultPartitionWriter) resultPartitionWriter).incrementResultCount();
                     frameOutputStream.reset();
                 }
             }


(asterixdb) 01/25: [ASTERIXDB-3635][COMP] Tune single dataset index costing

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 95d55701d4447ec79871c6c6106b537b5fd39bb0
Author: murali4104 <mu...@couchbase.com>
AuthorDate: Thu Sep 18 14:40:07 2025 -0700

    [ASTERIXDB-3635][COMP] Tune single dataset index costing
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Ext-Ref: MB-68074
    
    Change-Id: If2ed184e1dbc941ee53718e09331cbf7011370ae
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20393
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Integration-Tests: Preetham Poluparthi <pr...@apache.org>
    Reviewed-by: Preetham Poluparthi <pr...@apache.org>
    Tested-by: Preetham Poluparthi <pr...@apache.org>
---
 .../asterix/optimizer/rules/cbo/JoinNode.java      | 107 +++++++--------------
 .../single-collection-queries.3.plan               |  26 ++---
 .../single-collection-queries.4.plan               |  26 ++---
 .../single-collection-queries.7.plan               |  26 ++---
 4 files changed, 73 insertions(+), 112 deletions(-)

diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
index 5cbb6f80e7..c71009a23c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/JoinNode.java
@@ -730,99 +730,63 @@ public class JoinNode {
     private void buildIndexPlans() {
         List<PlanNode> allPlans = joinEnum.getAllPlans();
         PlanNode pn;
-        ICost opCost, totalCost;
+        double ic, dc, tc, oc; // for debugging
+
         List<Triple<Index, Double, AbstractFunctionCallExpression>> mandatoryIndexesInfo = new ArrayList<>();
         List<Triple<Index, Double, AbstractFunctionCallExpression>> optionalIndexesInfo = new ArrayList<>();
         double sel = 1.0;
-        boolean mandatoryArrayIndexUsed = false;
-        boolean optionalArrayIndexUsed = false;
-        opCost = this.joinEnum.getCostHandle().zeroCost();
+
         for (int i = 0; i < IndexCostInfo.size(); i++) {
             if (joinEnum.findUseIndexHint(IndexCostInfo.get(i).third)) {
                 mandatoryIndexesInfo.add(IndexCostInfo.get(i));
-                mandatoryArrayIndexUsed = mandatoryArrayIndexUsed
-                        || (mandatoryIndexesInfo.get(i).first.getIndexType() == DatasetConfig.IndexType.ARRAY);
             } else {
                 optionalIndexesInfo.add(IndexCostInfo.get(i));
-                optionalArrayIndexUsed = optionalArrayIndexUsed
-                        || (optionalIndexesInfo.get(i).first.getIndexType() == DatasetConfig.IndexType.ARRAY);
             }
         }
 
-        List<ICost> indexCosts = new ArrayList<>(); // these are the costs associated with the index only
-        // First cost all the mandatory indexes. These will be in the plan regardless of the cost
-        if (mandatoryIndexesInfo.size() > 0) {
-            for (int i = 0; i < mandatoryIndexesInfo.size(); i++) {
-                indexCosts.add(joinEnum.getCostMethodsHandle().costIndexScan(this, mandatoryIndexesInfo.get(i).second));
-            }
-
-            opCost = this.joinEnum.getCostHandle().zeroCost();
+        ICost indexCosts = this.joinEnum.getCostHandle().zeroCost();
+        ICost totalCost = this.joinEnum.getCostHandle().zeroCost();
+        ICost dataScanCost;
 
+        if (mandatoryIndexesInfo.size() > 0) {
             for (int i = 0; i < mandatoryIndexesInfo.size(); i++) {
-                opCost = opCost.costAdd(indexCosts.get(i)); // opCost will have all the index scan costs
-                sel *= mandatoryIndexesInfo.get(i).second; // assuming selectivities are independent for now
-            }
-
-            // Now add the data Scan cost.
-            ICost dataScanCost = joinEnum.getCostMethodsHandle().costIndexDataScan(this, sel);
-            opCost = opCost.costAdd(dataScanCost); // opCost now has the total cost of all the mandatory indexes + data costs.
-            if (mandatoryArrayIndexUsed) {
-                opCost = opCost.costAdd(opCost);
+                ICost cost = joinEnum.getCostMethodsHandle().costIndexScan(this, mandatoryIndexesInfo.get(i).second);
+                if ((mandatoryIndexesInfo.get(i).first.getIndexType() == DatasetConfig.IndexType.ARRAY)) {
+                    cost = cost.costAdd(cost); // double the cost for arrays.
+                }
+                indexCosts = indexCosts.costAdd(cost); // a running tally
+                sel *= mandatoryIndexesInfo.get(i).second;
             }
+            dataScanCost = joinEnum.getCostMethodsHandle().costIndexDataScan(this, sel);
+            totalCost = indexCosts.costAdd(dataScanCost); // this is the total cost of using the mandatory costs. This cost cannot be skipped.
         }
 
-        ICost mandatoryIndexesCost = opCost; // This will be added at the end to the total cost irrespective of optimality.
-        //opCost = this.joinEnum.getCostHandle().zeroCost(); // compute cost for optional indexes and store in opCost.
-        // Now lets deal with the optional indexes. These are the ones without any hints on them.
-        List<ICost> dataCosts = new ArrayList<>(); // these are the costs associated with accessing the data records
-        indexCosts.clear();
+        ICost opCost = totalCost;
         if (optionalIndexesInfo.size() > 0) {
-            optionalIndexesInfo.sort(Comparator.comparingDouble(o -> o.second)); // sort on selectivity.
-
-            // find the costs using one index at a time first.
+            optionalIndexesInfo.sort(Comparator.comparingDouble(o -> o.second));
 
-            // sel is now the selectivity of all the previous mandatory indexes.
             for (int i = 0; i < optionalIndexesInfo.size(); i++) {
-                indexCosts.add(joinEnum.getCostMethodsHandle().costIndexScan(this, optionalIndexesInfo.get(i).second)); // I0; I1; I2; ...
-                // Now get the cost of the datascans involved with the multiplied selectivity
-                // dataCost (0) will contain the dataScan cost with the first index
-                //dataCost (1) will contain the dataScan cost with the first index and the 2nd index and so on.
-                sel *= optionalIndexesInfo.get(i).second; // assuming selectivities are independent for now
-                if (optionalIndexesInfo.get(i).first.isPrimaryIndex()) {
-                    dataCosts.add(joinEnum.getCostHandle().zeroCost());
-                } else {
-                    dataCosts.add(joinEnum.getCostMethodsHandle().costIndexDataScan(this, sel)); // D0; D01; D012; ...
+                ICost cost = joinEnum.getCostMethodsHandle().costIndexScan(this, optionalIndexesInfo.get(i).second);
+                if ((optionalIndexesInfo.get(i).first.getIndexType() == DatasetConfig.IndexType.ARRAY)) {
+                    cost = cost.costAdd(cost); // double the cost for arrays.
                 }
-            }
-
-            // At the of of the above loop, I0, I1, I2 ... have been computed
-            // Also D0, D01, D012 ... have been computed.
-
-            opCost = indexCosts.get(0).costAdd(dataCosts.get(0));
-            //opCost is now the cost of the first (and cheapest) optional index plus the corresponding data scan
-
-            //Intersect the first two and then the first three and so on.
-            //If the cost does not decrease, then stop
-
-            ICost newIdxCost = indexCosts.get(0); // I0
-            ICost currentCost;
-            for (int i = 1; i < optionalIndexesInfo.size(); i++) {
-                newIdxCost = newIdxCost.costAdd(indexCosts.get(i)); // I0 + I1; I0 + I1 + I2
-                currentCost = newIdxCost.costAdd(dataCosts.get(i)); // I0 + I1 + D01; I0 + I1 + I2 + D012
-                if (currentCost.costLT(opCost) || level <= joinEnum.cboFullEnumLevel) { // save this cost and try adding one more index
-                    opCost = currentCost;
-                } else {
-                    // set the selectivites of the indexes not picked to be -1.0, so we can set
-                    // the skp index annotations correctly
-                    for (int j = i; j < optionalIndexesInfo.size(); j++) {
-                        optionalIndexesInfo.get(j).second = -1.0;
+                indexCosts = indexCosts.costAdd(cost);
+                sel *= optionalIndexesInfo.get(i).second;
+                dataScanCost = joinEnum.getCostMethodsHandle().costIndexDataScan(this, sel);
+                opCost = indexCosts.costAdd(dataScanCost);
+                tc = totalCost.computeTotalCost();
+                if (tc > 0.0) {
+                    if (opCost.costGT(totalCost)) { // we can stop here since additional indexes are not useful
+                        for (int j = i; j < optionalIndexesInfo.size(); j++) {
+                            optionalIndexesInfo.get(j).second = -1.0;
+                        }
+                        opCost = totalCost;
+                        break;
                     }
-                    break; // can't get any cheaper.
+                } else {
+                    totalCost = indexCosts.costAdd(dataScanCost);
                 }
             }
-            if (optionalArrayIndexUsed) {
-                opCost = opCost.costAdd(opCost);
-            }
         }
 
         // opCost is now the total cost of the indexes chosen along with the associated data scan cost.
@@ -832,9 +796,6 @@ public class JoinNode {
             }
         }
 
-        totalCost = opCost.costAdd(mandatoryIndexesCost); // cost of all the indexes chosen
-        // Now check if any of the indexes were array indexes. If so double the cost
-
         boolean forceEnum = mandatoryIndexesInfo.size() > 0 || level <= joinEnum.cboFullEnumLevel;
         if (opCost.costLT(this.cheapestPlanCost) || forceEnum) {
             pn = new PlanNode(allPlans.size(), joinEnum, this, datasetNames.get(0), leafInput);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.3.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.3.plan
index c6c8f6f3b4..bc53d0131a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.3.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.3.plan
@@ -1,38 +1,38 @@
-distribute result [$$48] [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+distribute result [$$48] [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
 -- DISTRIBUTE_RESULT  |PARTITIONED|
-  exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+  exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
   -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-    assign [$$48] <- [{"$1": $$51}] project: [$$48] [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+    assign [$$48] <- [{"$1": $$51}] project: [$$48] [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
     -- ASSIGN  |PARTITIONED|
-      project ([$$51]) [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+      project ([$$51]) [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
       -- STREAM_PROJECT  |PARTITIONED|
-        exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+        exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
         -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
           group by ([$$o_custkey := $$54]) decor ([]) {
                     aggregate [$$51] <- [sql-sum-serial($$53)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                     -- AGGREGATE  |LOCAL|
                       nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                       -- NESTED_TUPLE_SOURCE  |LOCAL|
-                 } [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4495.77]
+                 } [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 4497.18]
           -- EXTERNAL_GROUP_BY[$$54]  |PARTITIONED|
-            exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2997.18]
+            exchange [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2998.59]
             -- HASH_PARTITION_EXCHANGE [$$54]  |PARTITIONED|
               group by ([$$54 := $$49]) decor ([]) {
                         aggregate [$$53] <- [sql-count-serial(1)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                         -- AGGREGATE  |LOCAL|
                           nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- NESTED_TUPLE_SOURCE  |LOCAL|
-                     } [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2997.18]
+                     } [cardinality: 101.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2998.59]
               -- EXTERNAL_GROUP_BY[$$49]  |PARTITIONED|
-                exchange [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1498.59]
+                exchange [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                 -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                  assign [$$49] <- [$$o.getField(1)] project: [$$49] [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1498.59]
+                  assign [$$49] <- [$$o.getField(1)] project: [$$49] [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                   -- ASSIGN  |PARTITIONED|
-                    project ([$$o]) [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1498.59]
+                    project ([$$o]) [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                     -- STREAM_PROJECT  |PARTITIONED|
-                      exchange [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1498.59]
+                      exchange [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                       -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                        unnest-map [$$50, $$o] <- index-search("Orders", 0, "Default", "tpch", "Orders", false, false, 1, $$55, 0, false, true, false) [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1498.59]
+                        unnest-map [$$50, $$o] <- index-search("Orders", 0, "Default", "tpch", "Orders", false, false, 1, $$55, 0, false, true, false) [cardinality: 1498.59, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                         -- BTREE_SEARCH  |PARTITIONED|
                           exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.4.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.4.plan
index d725fcf9c5..5e8aa1ad14 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.4.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.4.plan
@@ -1,38 +1,38 @@
-distribute result [$$48] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+distribute result [$$48] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
 -- DISTRIBUTE_RESULT  |PARTITIONED|
-  exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+  exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
   -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-    assign [$$48] <- [{"$1": $$51}] project: [$$48] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+    assign [$$48] <- [{"$1": $$51}] project: [$$48] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
     -- ASSIGN  |PARTITIONED|
-      project ([$$51]) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+      project ([$$51]) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
       -- STREAM_PROJECT  |PARTITIONED|
-        exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+        exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
         -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
           group by ([$$o_orderdate := $$54]) decor ([]) {
                     aggregate [$$51] <- [sql-sum-serial($$53)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                     -- AGGREGATE  |LOCAL|
                       nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                       -- NESTED_TUPLE_SOURCE  |LOCAL|
-                 } [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4487.31]
+                 } [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 4491.54]
           -- EXTERNAL_GROUP_BY[$$54]  |PARTITIONED|
-            exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 2991.54]
+            exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 2995.77]
             -- HASH_PARTITION_EXCHANGE [$$54]  |PARTITIONED|
               group by ([$$54 := $$49]) decor ([]) {
                         aggregate [$$53] <- [sql-count-serial(1)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                         -- AGGREGATE  |LOCAL|
                           nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- NESTED_TUPLE_SOURCE  |LOCAL|
-                     } [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 2991.54]
+                     } [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 2995.77]
               -- EXTERNAL_GROUP_BY[$$49]  |PARTITIONED|
-                exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1495.77]
+                exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                 -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                  assign [$$49] <- [$$o.getField(4)] project: [$$49] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1495.77]
+                  assign [$$49] <- [$$o.getField(4)] project: [$$49] [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                   -- ASSIGN  |PARTITIONED|
-                    project ([$$o]) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1495.77]
+                    project ([$$o]) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                     -- STREAM_PROJECT  |PARTITIONED|
-                      exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1495.77]
+                      exchange [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                       -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                        unnest-map [$$50, $$o] <- index-search("Orders", 0, "Default", "tpch", "Orders", false, false, 1, $$55, 0, false, true, false) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1495.77]
+                        unnest-map [$$50, $$o] <- index-search("Orders", 0, "Default", "tpch", "Orders", false, false, 1, $$55, 0, false, true, false) [cardinality: 1495.77, doc-size: 0.0, op-cost: 0.0, total-cost: 1500.0]
                         -- BTREE_SEARCH  |PARTITIONED|
                           exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
index 7ec6db3b3c..cd177f77b9 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
@@ -1,38 +1,38 @@
-distribute result [$$51] [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+distribute result [$$51] [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
 -- DISTRIBUTE_RESULT  |PARTITIONED|
-  exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+  exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
   -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-    assign [$$51] <- [{"$1": $$55}] project: [$$51] [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+    assign [$$51] <- [{"$1": $$55}] project: [$$51] [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
     -- ASSIGN  |PARTITIONED|
-      project ([$$55]) [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+      project ([$$55]) [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
       -- STREAM_PROJECT  |PARTITIONED|
-        exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+        exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
         -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
           group by ([$$l_partkey := $$59]) decor ([]) {
                     aggregate [$$55] <- [sql-sum-serial($$58)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                     -- AGGREGATE  |LOCAL|
                       nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                       -- NESTED_TUPLE_SOURCE  |LOCAL|
-                 } [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 3965.67]
+                 } [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 8648.78]
           -- EXTERNAL_GROUP_BY[$$59]  |PARTITIONED|
-            exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2643.78]
+            exchange [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 7326.89]
             -- HASH_PARTITION_EXCHANGE [$$59]  |PARTITIONED|
               group by ([$$59 := $$52]) decor ([]) {
                         aggregate [$$58] <- [sql-count-serial(1)] [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                         -- AGGREGATE  |LOCAL|
                           nested tuple source [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- NESTED_TUPLE_SOURCE  |LOCAL|
-                     } [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 2643.78]
+                     } [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total-cost: 7326.89]
               -- EXTERNAL_GROUP_BY[$$52]  |PARTITIONED|
-                exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 1321.89]
+                exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                 -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                  select (and(gt($$53, 1), gt($$54, 4))) project: [$$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 1321.89]
+                  select (and(gt($$53, 1), gt($$54, 4))) project: [$$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                   -- STREAM_SELECT  |PARTITIONED|
-                    assign [$$52] <- [$$l.getField(1)] project: [$$53, $$54, $$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 1321.89]
+                    assign [$$52] <- [$$l.getField(1)] project: [$$53, $$54, $$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                     -- ASSIGN  |PARTITIONED|
-                      exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 1321.89]
+                      exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                       -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                        unnest-map [$$53, $$54, $$l] <- index-search("LineItem", 0, "Default", "tpch", "LineItem", false, false, 2, $$60, $$61, 0, true, true, false) [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 1321.89]
+                        unnest-map [$$53, $$54, $$l] <- index-search("LineItem", 0, "Default", "tpch", "LineItem", false, false, 2, $$60, $$61, 0, true, true, false) [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                         -- BTREE_SEARCH  |PARTITIONED|
                           exchange [cardinality: 0.0, doc-size: 0.0, op-cost: 0.0, total-cost: 0.0]
                           -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|


(asterixdb) 09/25: [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit b20e3c66d9a8a9c695f0314060aadd43954e47a2
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Oct 2 18:14:09 2025 -0400

    [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
    
    ...when flushPtr is also that page
    
    (cherry picked from commit 75dd626ac1)
    
    Ext-ref:MB-68818
    Change-Id: I1a710fa4773b2d75ff4f252399f4dadccdf3ed8d
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20471
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
    Tested-by: Michael Blow <mb...@apache.org>
---
 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 +++++++++++++++++++++
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 2 files changed, 103 insertions(+), 3 deletions(-)

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
index b056bbe162..0a1fba2020 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.test.dataflow;
 
 import java.io.File;
+import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -32,6 +33,7 @@ import org.apache.asterix.app.nc.NCAppRuntimeContext;
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.StorageProperties.Option;
+import org.apache.asterix.common.context.GlobalVirtualBufferCache;
 import org.apache.asterix.common.metadata.DataverseName;
 import org.apache.asterix.common.transactions.ITransactionContext;
 import org.apache.asterix.common.transactions.ITransactionManager;
@@ -60,6 +62,7 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -175,6 +178,103 @@ public class GlobalVirtualBufferCacheTest {
         }
     }
 
+    @Test
+    public void testFlushPtrBoundsAfterRemovingLastIndex() throws Exception {
+        try {
+            // Create a scenario with multiple indexes to manipulate flushPtr
+            DataverseName dvName = DataverseName.createSinglePartName(StorageTestUtils.DATAVERSE_NAME);
+
+            // Create 3 additional datasets to have multiple indexes
+            Dataset dataset2 = new TestDataset(dvName, "ds2", dvName, StorageTestUtils.DATA_TYPE_NAME,
+                    StorageTestUtils.NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null,
+                    new InternalDatasetDetails(null, PartitioningStrategy.HASH, StorageTestUtils.PARTITIONING_KEYS,
+                            null, null, null, false, null, null),
+                    null, DatasetType.INTERNAL, StorageTestUtils.DATASET_ID + 2, 0);
+
+            Dataset dataset3 = new TestDataset(dvName, "ds3", dvName, StorageTestUtils.DATA_TYPE_NAME,
+                    StorageTestUtils.NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null,
+                    new InternalDatasetDetails(null, PartitioningStrategy.HASH, StorageTestUtils.PARTITIONING_KEYS,
+                            null, null, null, false, null, null),
+                    null, DatasetType.INTERNAL, StorageTestUtils.DATASET_ID + 3, 0);
+
+            // Create indexes for first partition only for simplicity
+            PrimaryIndexInfo index2Info = StorageTestUtils.createPrimaryIndex(nc, dataset2, 0);
+            PrimaryIndexInfo index3Info = StorageTestUtils.createPrimaryIndex(nc, dataset3, 0);
+
+            // Create index instances
+            IIndexDataflowHelperFactory factory2 =
+                    new IndexDataflowHelperFactory(nc.getStorageManager(), index2Info.getFileSplitProvider());
+            IIndexDataflowHelper helper2 = factory2.create(testCtxs[0].getJobletContext().getServiceContext(), 0);
+            helper2.open();
+            TestLsmBtree index2 = (TestLsmBtree) helper2.getIndexInstance();
+            helper2.close();
+
+            IIndexDataflowHelperFactory factory3 =
+                    new IndexDataflowHelperFactory(nc.getStorageManager(), index3Info.getFileSplitProvider());
+            IIndexDataflowHelper helper3 = factory3.create(testCtxs[0].getJobletContext().getServiceContext(), 0);
+            helper3.open();
+            TestLsmBtree index3 = (TestLsmBtree) helper3.getIndexInstance();
+            helper3.close();
+
+            // Access the global VBC through reflection to manipulate flushPtr
+            GlobalVirtualBufferCache globalVBC = (GlobalVirtualBufferCache) ncAppCtx.getVirtualBufferCache();
+
+            // Use reflection to access private fields
+            Field flushPtrField = GlobalVirtualBufferCache.class.getDeclaredField("flushPtr");
+            flushPtrField.setAccessible(true);
+
+            Field primaryIndexesField = GlobalVirtualBufferCache.class.getDeclaredField("primaryIndexes");
+            primaryIndexesField.setAccessible(true);
+
+            // Register memory components to add indexes to the list
+            // We already have primaryIndexes[0] and filteredPrimaryIndexes[0] registered
+            globalVBC.register(index2.getCurrentMemoryComponent());
+            globalVBC.register(index3.getCurrentMemoryComponent());
+
+            // Get the current state
+            @SuppressWarnings("unchecked")
+            List<ILSMIndex> primaryIndexes = (List<ILSMIndex>) primaryIndexesField.get(globalVBC);
+            int currentSize = primaryIndexes.size();
+
+            // Set flushPtr to point to the last index (should be currentSize - 1)
+            flushPtrField.set(globalVBC, currentSize - 1);
+            int flushPtrBefore = (Integer) flushPtrField.get(globalVBC);
+
+            // Verify we have the expected setup
+            Assert.assertTrue("Should have at least 3 indexes", currentSize >= 3);
+            Assert.assertEquals("flushPtr should point to last index", currentSize - 1, flushPtrBefore);
+
+            // Remove the last index (the one flushPtr is pointing to)
+            ILSMIndex lastIndex = primaryIndexes.get(currentSize - 1);
+            globalVBC.unregister(lastIndex.getCurrentMemoryComponent());
+
+            // Check the state after removal
+            int newSize = primaryIndexes.size();
+            int flushPtrAfter = (Integer) flushPtrField.get(globalVBC);
+
+            // This demonstrates the bug: flushPtr should be within bounds [0, newSize)
+            // With the current buggy code (flushPtr > pos), flushPtr would be (currentSize-1-1) % newSize
+            // But if flushPtr == pos (pointing to removed index), it's not adjusted and points out of bounds
+
+            Assert.assertEquals("List should be smaller after removal", currentSize - 1, newSize);
+
+            // The bug manifests when flushPtr equals the position of the removed index
+            // In this case, flushPtr is not adjusted and points beyond the list
+            if (flushPtrBefore == currentSize - 1) { // We removed the index flushPtr was pointing to
+                Assert.assertTrue("flushPtr should be within bounds after removal",
+                        flushPtrAfter >= 0 && flushPtrAfter < newSize);
+            }
+
+            // Clean up
+            helper2.destroy();
+            helper3.destroy();
+
+        } catch (Exception e) {
+            LOGGER.error("testFlushPtrBoundsAfterRemovingLastIndex failed", e);
+            Assert.fail(e.getMessage());
+        }
+    }
+
     private void initializeNc() throws Exception {
         List<Pair<IOption, Object>> opts = new ArrayList<>();
         opts.add(Pair.of(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET, 128 * 1024L));
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index f8771430e4..12c660da40 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -74,7 +74,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
 
     private final Set<ILSMIndex> flushingIndexes = Collections.synchronizedSet(new HashSet<>());
     private final Set<ILSMMemoryComponent> flushingComponents = Collections.synchronizedSet(new HashSet<>());
-    private volatile int flushPtr;
+    private int flushPtr;
 
     private final int filteredMemoryComponentMaxNumPages;
     private final int flushPageBudget;
@@ -87,7 +87,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
         this.vbc = new VirtualBufferCache(allocator, storageProperties.getBufferCachePageSize(),
                 (int) (storageProperties.getMemoryComponentGlobalBudget()
                         / storageProperties.getMemoryComponentPageSize()));
-        this.flushPageBudget = (int) (storageProperties.getMemoryComponentGlobalBudget()
+        this.flushPageBudget = (int) ((double) storageProperties.getMemoryComponentGlobalBudget()
                 / storageProperties.getMemoryComponentPageSize()
                 * storageProperties.getMemoryComponentFlushThreshold());
         this.filteredMemoryComponentMaxNumPages = storageProperties.getFilteredMemoryComponentMaxNumPages();
@@ -145,7 +145,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
                     }
                     if (primaryIndexes.isEmpty()) {
                         flushPtr = 0;
-                    } else if (flushPtr > pos) {
+                    } else if (flushPtr >= pos) {
                         // If the removed index is before flushPtr, we should decrement flushPtr by 1 so that
                         // it still points to the same index.
                         flushPtr = (flushPtr - 1) % primaryIndexes.size();


(asterixdb) 02/25: [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 75dd626ac1885d4ae9c63f2cbff845a6d5e3ae19
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Oct 2 18:14:09 2025 -0400

    [NO ISSUE][*DB][STO] Fix IndexOutOfBoundsException removing last page...
    
    ...when flushPtr is also that page
    
    Ext-ref:MB-68818
    Change-Id: I1a710fa4773b2d75ff4f252399f4dadccdf3ed8d
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20439
    Reviewed-by: Michael Blow <mb...@apache.org>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Tested-by: Michael Blow <mb...@apache.org>
---
 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 +++++++++++++++++++++
 .../lsm/flush/FlushColumnTupleWriter.java          |  14 +--
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 3 files changed, 111 insertions(+), 9 deletions(-)

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
index b056bbe162..0a1fba2020 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/GlobalVirtualBufferCacheTest.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.test.dataflow;
 
 import java.io.File;
+import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -32,6 +33,7 @@ import org.apache.asterix.app.nc.NCAppRuntimeContext;
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.StorageProperties.Option;
+import org.apache.asterix.common.context.GlobalVirtualBufferCache;
 import org.apache.asterix.common.metadata.DataverseName;
 import org.apache.asterix.common.transactions.ITransactionContext;
 import org.apache.asterix.common.transactions.ITransactionManager;
@@ -60,6 +62,7 @@ import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.impls.AbstractTreeIndex;
 import org.apache.hyracks.storage.am.lsm.btree.impl.TestLsmBtree;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -175,6 +178,103 @@ public class GlobalVirtualBufferCacheTest {
         }
     }
 
+    @Test
+    public void testFlushPtrBoundsAfterRemovingLastIndex() throws Exception {
+        try {
+            // Create a scenario with multiple indexes to manipulate flushPtr
+            DataverseName dvName = DataverseName.createSinglePartName(StorageTestUtils.DATAVERSE_NAME);
+
+            // Create 3 additional datasets to have multiple indexes
+            Dataset dataset2 = new TestDataset(dvName, "ds2", dvName, StorageTestUtils.DATA_TYPE_NAME,
+                    StorageTestUtils.NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null,
+                    new InternalDatasetDetails(null, PartitioningStrategy.HASH, StorageTestUtils.PARTITIONING_KEYS,
+                            null, null, null, false, null, null),
+                    null, DatasetType.INTERNAL, StorageTestUtils.DATASET_ID + 2, 0);
+
+            Dataset dataset3 = new TestDataset(dvName, "ds3", dvName, StorageTestUtils.DATA_TYPE_NAME,
+                    StorageTestUtils.NODE_GROUP_NAME, NoMergePolicyFactory.NAME, null,
+                    new InternalDatasetDetails(null, PartitioningStrategy.HASH, StorageTestUtils.PARTITIONING_KEYS,
+                            null, null, null, false, null, null),
+                    null, DatasetType.INTERNAL, StorageTestUtils.DATASET_ID + 3, 0);
+
+            // Create indexes for first partition only for simplicity
+            PrimaryIndexInfo index2Info = StorageTestUtils.createPrimaryIndex(nc, dataset2, 0);
+            PrimaryIndexInfo index3Info = StorageTestUtils.createPrimaryIndex(nc, dataset3, 0);
+
+            // Create index instances
+            IIndexDataflowHelperFactory factory2 =
+                    new IndexDataflowHelperFactory(nc.getStorageManager(), index2Info.getFileSplitProvider());
+            IIndexDataflowHelper helper2 = factory2.create(testCtxs[0].getJobletContext().getServiceContext(), 0);
+            helper2.open();
+            TestLsmBtree index2 = (TestLsmBtree) helper2.getIndexInstance();
+            helper2.close();
+
+            IIndexDataflowHelperFactory factory3 =
+                    new IndexDataflowHelperFactory(nc.getStorageManager(), index3Info.getFileSplitProvider());
+            IIndexDataflowHelper helper3 = factory3.create(testCtxs[0].getJobletContext().getServiceContext(), 0);
+            helper3.open();
+            TestLsmBtree index3 = (TestLsmBtree) helper3.getIndexInstance();
+            helper3.close();
+
+            // Access the global VBC through reflection to manipulate flushPtr
+            GlobalVirtualBufferCache globalVBC = (GlobalVirtualBufferCache) ncAppCtx.getVirtualBufferCache();
+
+            // Use reflection to access private fields
+            Field flushPtrField = GlobalVirtualBufferCache.class.getDeclaredField("flushPtr");
+            flushPtrField.setAccessible(true);
+
+            Field primaryIndexesField = GlobalVirtualBufferCache.class.getDeclaredField("primaryIndexes");
+            primaryIndexesField.setAccessible(true);
+
+            // Register memory components to add indexes to the list
+            // We already have primaryIndexes[0] and filteredPrimaryIndexes[0] registered
+            globalVBC.register(index2.getCurrentMemoryComponent());
+            globalVBC.register(index3.getCurrentMemoryComponent());
+
+            // Get the current state
+            @SuppressWarnings("unchecked")
+            List<ILSMIndex> primaryIndexes = (List<ILSMIndex>) primaryIndexesField.get(globalVBC);
+            int currentSize = primaryIndexes.size();
+
+            // Set flushPtr to point to the last index (should be currentSize - 1)
+            flushPtrField.set(globalVBC, currentSize - 1);
+            int flushPtrBefore = (Integer) flushPtrField.get(globalVBC);
+
+            // Verify we have the expected setup
+            Assert.assertTrue("Should have at least 3 indexes", currentSize >= 3);
+            Assert.assertEquals("flushPtr should point to last index", currentSize - 1, flushPtrBefore);
+
+            // Remove the last index (the one flushPtr is pointing to)
+            ILSMIndex lastIndex = primaryIndexes.get(currentSize - 1);
+            globalVBC.unregister(lastIndex.getCurrentMemoryComponent());
+
+            // Check the state after removal
+            int newSize = primaryIndexes.size();
+            int flushPtrAfter = (Integer) flushPtrField.get(globalVBC);
+
+            // This demonstrates the bug: flushPtr should be within bounds [0, newSize)
+            // With the current buggy code (flushPtr > pos), flushPtr would be (currentSize-1-1) % newSize
+            // But if flushPtr == pos (pointing to removed index), it's not adjusted and points out of bounds
+
+            Assert.assertEquals("List should be smaller after removal", currentSize - 1, newSize);
+
+            // The bug manifests when flushPtr equals the position of the removed index
+            // In this case, flushPtr is not adjusted and points beyond the list
+            if (flushPtrBefore == currentSize - 1) { // We removed the index flushPtr was pointing to
+                Assert.assertTrue("flushPtr should be within bounds after removal",
+                        flushPtrAfter >= 0 && flushPtrAfter < newSize);
+            }
+
+            // Clean up
+            helper2.destroy();
+            helper3.destroy();
+
+        } catch (Exception e) {
+            LOGGER.error("testFlushPtrBoundsAfterRemovingLastIndex failed", e);
+            Assert.fail(e.getMessage());
+        }
+    }
+
     private void initializeNc() throws Exception {
         List<Pair<IOption, Object>> opts = new ArrayList<>();
         opts.add(Pair.of(Option.STORAGE_MEMORYCOMPONENT_GLOBALBUDGET, 128 * 1024L));
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
index 56149bc205..27765c0508 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/operation/lsm/flush/FlushColumnTupleWriter.java
@@ -245,12 +245,14 @@ public class FlushColumnTupleWriter extends AbstractColumnTupleWriter {
         finalizer.finalizeBatchColumns(columnMetadata, presentColumnsIndexes, pageZeroWriter);
 
         //assertion logging
-        int presentColumnsCount = presentColumnsIndexes.cardinality();
-        int beforeTransformColumnCount = transformerForCurrentTuple.getBeforeTransformColumnsCount();
-        int currentTupleColumnsCount = transformerForCurrentTuple.getNumberOfVisitedColumnsInBatch();
-        if (beforeTransformColumnCount != presentColumnsCount || currentTupleColumnsCount != presentColumnsCount) {
-            LOGGER.debug("mismatch in column counts: beforeTransform={}, currentTuple={}, expected={}",
-                    beforeTransformColumnCount, currentTupleColumnsCount, presentColumnsCount);
+        if (LOGGER.isTraceEnabled()) {
+            int presentColumnsCount = presentColumnsIndexes.cardinality();
+            int beforeTransformColumnCount = transformerForCurrentTuple.getBeforeTransformColumnsCount();
+            int currentTupleColumnsCount = transformerForCurrentTuple.getNumberOfVisitedColumnsInBatch();
+            if (beforeTransformColumnCount != presentColumnsCount || currentTupleColumnsCount != presentColumnsCount) {
+                LOGGER.trace("mismatch in column counts: beforeTransform={}, currentTuple={}, expected={}",
+                        beforeTransformColumnCount, currentTupleColumnsCount, presentColumnsCount);
+            }
         }
 
         writer.setPageZeroWriter(pageZeroWriter, toIndexArray(presentColumnsIndexes), numberOfColumns);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index cb5323fd15..2c925fd028 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -78,7 +78,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
 
     private final Set<ILSMIndex> flushingIndexes = Collections.synchronizedSet(new HashSet<>());
     private final Set<ILSMMemoryComponent> flushingComponents = Collections.synchronizedSet(new HashSet<>());
-    private volatile int flushPtr;
+    private int flushPtr;
 
     private final int filteredMemoryComponentMaxNumPages;
     private final int flushPageBudget;
@@ -91,7 +91,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
         this.vbc = new VirtualBufferCache(allocator, storageProperties.getBufferCachePageSize(),
                 (int) (storageProperties.getMemoryComponentGlobalBudget()
                         / storageProperties.getMemoryComponentPageSize()));
-        this.flushPageBudget = (int) (storageProperties.getMemoryComponentGlobalBudget()
+        this.flushPageBudget = (int) ((double) storageProperties.getMemoryComponentGlobalBudget()
                 / storageProperties.getMemoryComponentPageSize()
                 * storageProperties.getMemoryComponentFlushThreshold());
         this.filteredMemoryComponentMaxNumPages = storageProperties.getFilteredMemoryComponentMaxNumPages();
@@ -149,7 +149,7 @@ public class GlobalVirtualBufferCache implements IVirtualBufferCache, ILifeCycle
                     }
                     if (primaryIndexes.isEmpty()) {
                         flushPtr = 0;
-                    } else if (flushPtr > pos) {
+                    } else if (flushPtr >= pos) {
                         // If the removed index is before flushPtr, we should decrement flushPtr by 1 so that
                         // it still points to the same index.
                         flushPtr = (flushPtr - 1) % primaryIndexes.size();


(asterixdb) 19/25: [NO ISSUE][CLOUD] Updating deleteObjects for AzureBlob

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 4cb134ee3d4ca05a13b6f2df42e6643c2d598444
Author: Ritik Raj <ra...@gmail.com>
AuthorDate: Tue Oct 21 10:02:34 2025 +0530

    [NO ISSUE][CLOUD] Updating deleteObjects for AzureBlob
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    Replacing the expensive LIST call while bulk deleting
    with the async delete calls
    
    Ext-ref: MB-68999
    Change-Id: Ie66fe1c9d95684af0c79b9387194f4bba01891a6
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20507
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
---
 .../blobstorage/AzBlobStorageClientConfig.java     |  2 +-
 .../blobstorage/AzBlobStorageCloudClient.java      | 97 ++++++++--------------
 2 files changed, 34 insertions(+), 65 deletions(-)

diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
index 889a6a42d8..31b160d1f5 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageClientConfig.java
@@ -30,7 +30,7 @@ import com.azure.identity.DefaultAzureCredentialBuilder;
 
 public class AzBlobStorageClientConfig {
     // Ref: https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id
-    static final int DELETE_BATCH_SIZE = 256;
+    static final int MAX_CONCURRENT_REQUESTS = 20;
 
     private final int writeBufferSize;
     private final String region;
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index 4a61f1caa9..0bd16a867b 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -19,7 +19,7 @@
 
 package org.apache.asterix.cloud.clients.azure.blobstorage;
 
-import static org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageClientConfig.DELETE_BATCH_SIZE;
+import static org.apache.asterix.cloud.clients.azure.blobstorage.AzBlobStorageClientConfig.MAX_CONCURRENT_REQUESTS;
 
 import java.io.ByteArrayOutputStream;
 import java.io.FilenameFilter;
@@ -31,7 +31,6 @@ import java.nio.ReadOnlyBufferException;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Iterator;
 import java.util.List;
 import java.util.Set;
 import java.util.function.Predicate;
@@ -60,14 +59,14 @@ import org.apache.logging.log4j.Logger;
 
 import com.azure.core.http.netty.NettyAsyncHttpClientBuilder;
 import com.azure.core.http.rest.PagedIterable;
-import com.azure.core.http.rest.Response;
 import com.azure.core.util.BinaryData;
+import com.azure.storage.blob.BlobAsyncClient;
 import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerAsyncClient;
 import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceAsyncClient;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
-import com.azure.storage.blob.batch.BlobBatchClient;
-import com.azure.storage.blob.batch.BlobBatchClientBuilder;
 import com.azure.storage.blob.models.BlobErrorCode;
 import com.azure.storage.blob.models.BlobItem;
 import com.azure.storage.blob.models.BlobListDetails;
@@ -83,6 +82,8 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.netty.handler.ssl.SslContext;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
 import reactor.netty.http.client.HttpClient;
 
 public class AzBlobStorageCloudClient implements ICloudClient {
@@ -91,12 +92,11 @@ public class AzBlobStorageCloudClient implements ICloudClient {
     private static final String AZURITE_ACCOUNT_NAME = "devstoreaccount1";
     private static final String AZURITE_ACCOUNT_KEY =
             "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
-    private static final int SUCCESS_RESPONSE_CODE = 202;
     private final ICloudGuardian guardian;
     private final BlobContainerClient blobContainerClient;
+    private final BlobContainerAsyncClient blobContainerAsyncClient;
     private final AzBlobStorageClientConfig config;
     private final IRequestProfilerLimiter profiler;
-    private final BlobBatchClient blobBatchClient;
     private static final Logger LOGGER = LogManager.getLogger();
 
     public AzBlobStorageCloudClient(AzBlobStorageClientConfig config, ICloudGuardian guardian) {
@@ -106,6 +106,7 @@ public class AzBlobStorageCloudClient implements ICloudClient {
     public AzBlobStorageCloudClient(AzBlobStorageClientConfig config, BlobServiceClient blobServiceClient,
             ICloudGuardian guardian) {
         this.blobContainerClient = blobServiceClient.getBlobContainerClient(config.getBucket());
+        this.blobContainerAsyncClient = buildAsyncClient(config).getBlobContainerAsyncClient(config.getBucket());
         this.config = config;
         this.guardian = guardian;
         long profilerInterval = config.getProfilerLogInterval();
@@ -116,7 +117,6 @@ public class AzBlobStorageCloudClient implements ICloudClient {
             profiler = new RequestLimiterNoOpProfiler(limiter);
         }
         guardian.setCloudClient(this);
-        blobBatchClient = new BlobBatchClientBuilder(blobContainerClient.getServiceClient()).buildClient();
     }
 
     @Override
@@ -277,59 +277,25 @@ public class AzBlobStorageCloudClient implements ICloudClient {
     public void deleteObjects(String bucket, Collection<String> paths) throws HyracksDataException {
         if (paths.isEmpty())
             return;
-        Set<BlobItem> blobsToDelete = getBlobsMatchingThesePaths(paths);
-        List<String> blobURLs = getBlobURLs(blobsToDelete);
-        if (blobURLs.isEmpty())
-            return;
-        Collection<List<String>> batchedBlobURLs = getBatchedBlobURLs(blobURLs);
-        for (List<String> batch : batchedBlobURLs) {
-            PagedIterable<Response<Void>> responses = blobBatchClient.deleteBlobs(batch, null);
-            Iterator<String> deletePathIter = paths.iterator();
-            String deletedPath;
-            String failedDeletedPath = null;
-            for (Response<Void> response : responses) {
-                deletedPath = deletePathIter.next();
-                // The response.getStatusCode() method returns:
-                // - 202 (Accepted) if the delete operation is successful
-                // - exception if the delete operation fails
-                int statusCode = response.getStatusCode();
-                if (statusCode != SUCCESS_RESPONSE_CODE) {
-                    LOGGER.warn("Failed to delete blob: {} with status code: {} while deleting {}", deletedPath,
-                            statusCode, paths.toString());
-                    if (failedDeletedPath == null) {
-                        failedDeletedPath = deletedPath;
-                    }
-                }
-            }
-            if (failedDeletedPath != null) {
-                throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, "DELETE", failedDeletedPath,
-                        paths.toString());
+
+        List<Mono<Boolean>> deleteMonos = new ArrayList<>();
+        for (String path : paths) {
+            if (path != null && !path.isEmpty()) {
+                BlobAsyncClient blobAsyncClient =
+                        blobContainerAsyncClient.getBlobAsyncClient(config.getPrefix() + path);
+                deleteMonos.add(blobAsyncClient.deleteIfExists());
             }
         }
-    }
 
-    private Collection<List<String>> getBatchedBlobURLs(List<String> blobURLs) {
-        int startIdx = 0;
-        Collection<List<String>> batchedBLOBURLs = new ArrayList<>();
-        Iterator<String> iterator = blobURLs.iterator();
-        while (iterator.hasNext()) {
-            List<String> batch = new ArrayList<>();
-            while (startIdx < DELETE_BATCH_SIZE && iterator.hasNext()) {
-                batch.add(iterator.next());
-                startIdx++;
-            }
-            batchedBLOBURLs.add(batch);
-            startIdx = 0;
+        if (deleteMonos.isEmpty()) {
+            return;
         }
-        return batchedBLOBURLs;
-    }
 
-    private Set<BlobItem> getBlobsMatchingThesePaths(Collection<String> paths) {
-        List<String> pathWithPrefix =
-                paths.stream().map(path -> config.getPrefix() + path).collect(Collectors.toList());
-        PagedIterable<BlobItem> blobItems = blobContainerClient.listBlobs();
-        return blobItems.stream().filter(blobItem -> pathWithPrefix.contains(blobItem.getName()))
-                .collect(Collectors.toSet());
+        try {
+            Flux.fromIterable(deleteMonos).flatMap(mono -> mono, MAX_CONCURRENT_REQUESTS).then().block();
+        } catch (Exception ex) {
+            throw new RuntimeDataException(ErrorCode.CLOUD_IO_FAILURE, "DELETE", ex, paths.toString());
+        }
     }
 
     @Override
@@ -423,6 +389,16 @@ public class AzBlobStorageCloudClient implements ICloudClient {
     }
 
     private static BlobServiceClient buildClient(AzBlobStorageClientConfig config) {
+        BlobServiceClientBuilder blobServiceClientBuilder = getBlobServiceClientBuilder(config);
+        return blobServiceClientBuilder.buildClient();
+    }
+
+    private static BlobServiceAsyncClient buildAsyncClient(AzBlobStorageClientConfig config) {
+        BlobServiceClientBuilder blobServiceClientBuilder = getBlobServiceClientBuilder(config);
+        return blobServiceClientBuilder.buildAsyncClient();
+    }
+
+    private static BlobServiceClientBuilder getBlobServiceClientBuilder(AzBlobStorageClientConfig config) {
         BlobServiceClientBuilder blobServiceClientBuilder = new BlobServiceClientBuilder();
         blobServiceClientBuilder.endpoint(getEndpoint(config));
         blobServiceClientBuilder.httpLogOptions(AzureConstants.HTTP_LOG_OPTIONS);
@@ -444,8 +420,7 @@ public class AzBlobStorageCloudClient implements ICloudClient {
                 throw new RuntimeException("Failed to disable SSL verification", e);
             }
         }
-
-        return blobServiceClientBuilder.buildClient();
+        return blobServiceClientBuilder;
     }
 
     private static void configCredentialsToAzClient(BlobServiceClientBuilder builder,
@@ -468,10 +443,4 @@ public class AzBlobStorageCloudClient implements ICloudClient {
         return config.isAnonymousAuth() ? AZURITE_ENDPOINT + config.getBucket()
                 : config.getEndpoint() + "/" + config.getBucket();
     }
-
-    private List<String> getBlobURLs(Set<BlobItem> blobs) {
-        final String blobURLPrefix = blobContainerClient.getBlobContainerUrl() + "/";
-        return blobs.stream().map(BlobItem::getName).map(blobName -> blobURLPrefix + blobName)
-                .collect(Collectors.toList());
-    }
 }


(asterixdb) 16/25: [NO ISSUE][RT] Respond to NC in case of unexpected failures

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 4e0194dc1ff8bd809ada2ddb52523b3789f8efd5
Author: Murtadha Hubail <mh...@apache.org>
AuthorDate: Fri Oct 17 15:49:31 2025 +0300

    [NO ISSUE][RT] Respond to NC in case of unexpected failures
    
    - user model changes: no
    - storage format changes: no
    - interface changes: yes
    
    Details:
    
    - On unexpected failures while sending messages from CC to NC,
      send a failure response to the NC to avoid requests hanging
      on the NC until their timeout.
    
    Ext-ref: MB-68985
    Change-Id: I5ff53205d7cbb23c91a482b22165e1645247745f
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20496
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Tested-by: Murtadha Hubail <mh...@apache.org>
---
 .../message/AbstractInternalRequestMessage.java    |  2 ++
 .../message/ExecuteStatementRequestMessage.java    | 32 ++++++++++++++--------
 .../apache/asterix/messaging/CCMessageBroker.java  |  9 ++++++
 .../common/messaging/api/ICCMessageBroker.java     |  8 ++++++
 4 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractInternalRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractInternalRequestMessage.java
index 7d58e4e2e8..41c4906f84 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractInternalRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/AbstractInternalRequestMessage.java
@@ -113,6 +113,8 @@ public abstract class AbstractInternalRequestMessage implements ICcAddressedMess
             messageBroker.sendApplicationMessageToNC(responseMsg, nodeRequestId);
         } catch (Exception e) {
             LOGGER.log(Level.WARN, e.toString(), e);
+            responseMsg.setError(new Exception(e.getMessage()));
+            messageBroker.sendMessageQuietly(responseMsg, nodeRequestId);
         }
 
     }
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 0cdbf95f25..94d6309d55 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -143,7 +143,7 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
         CCMessageBroker messageBroker = (CCMessageBroker) ccSrvContext.getMessageBroker();
         final RuntimeDataException rejectionReason = getRejectionReason(ccSrv, requestNodeId);
         if (rejectionReason != null) {
-            sendRejection(rejectionReason, messageBroker, requestMessageId, requestNodeId);
+            sendRejection(rejectionReason, messageBroker);
             return;
         }
         CCExtensionManager ccExtMgr = (CCExtensionManager) ccAppCtx.getExtensionManager();
@@ -195,11 +195,7 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
             GlobalConfig.ASTERIX_LOGGER.log(Level.ERROR, "Unexpected exception", e);
             responseMsg.setError(e);
         }
-        try {
-            messageBroker.sendApplicationMessageToNC(responseMsg, requestNodeId);
-        } catch (Exception e) {
-            LOGGER.log(Level.WARN, e.toString(), e);
-        }
+        sendResponseToNc(messageBroker, responseMsg);
     }
 
     protected IRequestParameters createRequestParameters(IStatementExecutor.StatementProperties statementProperties,
@@ -231,11 +227,8 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
         return null;
     }
 
-    protected void sendRejection(Exception reason, CCMessageBroker messageBroker, long requestMessageId,
-            String requestNodeId) {
-        ExecuteStatementResponseMessage responseMsg =
-                new ExecuteStatementResponseMessage(requestMessageId, clientContextID, requestReference.getUuid());
-        responseMsg.setError(reason);
+    protected void sendRejection(Exception reason, CCMessageBroker messageBroker) {
+        ExecuteStatementResponseMessage responseMsg = createFailureMessage(reason);
         try {
             messageBroker.sendApplicationMessageToNC(responseMsg, requestNodeId);
         } catch (Exception e) {
@@ -243,6 +236,23 @@ public class ExecuteStatementRequestMessage implements ICcAddressedMessage {
         }
     }
 
+    private void sendResponseToNc(CCMessageBroker messageBroker, ExecuteStatementResponseMessage responseMsg) {
+        try {
+            messageBroker.sendApplicationMessageToNC(responseMsg, requestNodeId);
+        } catch (Exception e) {
+            LOGGER.error("unexpected exception sending message to node {}", requestNodeId, e);
+            ExecuteStatementResponseMessage failureMessage = createFailureMessage(new Exception(e.getMessage()));
+            messageBroker.sendMessageQuietly(failureMessage, requestNodeId);
+        }
+    }
+
+    private ExecuteStatementResponseMessage createFailureMessage(Exception reason) {
+        ExecuteStatementResponseMessage responseMsg =
+                new ExecuteStatementResponseMessage(requestMessageId, clientContextID, requestReference.getUuid());
+        responseMsg.setError(reason);
+        return responseMsg;
+    }
+
     @Override
     public String toString() {
         if (statementsText != null && (statementsText.startsWith("UPSERT") || statementsText.startsWith("INSERT"))) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
index 06832074c9..eb538f0681 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/messaging/CCMessageBroker.java
@@ -69,6 +69,15 @@ public class CCMessageBroker implements ICCMessageBroker {
         return sendMessage(msg, nodeId, false);
     }
 
+    @Override
+    public void sendMessageQuietly(INcAddressedMessage msg, String nodeId) {
+        try {
+            sendMessage(msg, nodeId, false);
+        } catch (Exception e) {
+            LOGGER.warn("failed to send message to node {}", nodeId, e);
+        }
+    }
+
     @Override
     public boolean sendRealTimeApplicationMessageToNC(INcAddressedMessage msg, String nodeId) throws Exception {
         return sendMessage(msg, nodeId, true);
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/ICCMessageBroker.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/ICCMessageBroker.java
index e628bc7b41..72b0040243 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/ICCMessageBroker.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/messaging/api/ICCMessageBroker.java
@@ -38,6 +38,14 @@ public interface ICCMessageBroker extends IMessageBroker {
      */
     boolean sendApplicationMessageToNC(INcAddressedMessage msg, String nodeId) throws Exception;
 
+    /**
+     * Attempts to send {@code msg} to the specified {@code nodeId} without throwing exceptions
+     *
+     * @param msg
+     * @param nodeId
+     */
+    void sendMessageQuietly(INcAddressedMessage msg, String nodeId);
+
     /**
      * Sends the passed message to the specified {@code nodeId}
      *


(asterixdb) 25/25: Merge branch 'gerrit/phoenix' into 'master'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 362f97b98aeeed5603965e29f87ea0d163e95828
Merge: f388dbc8d9 39ac77d88a
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Thu Oct 23 20:14:26 2025 +0530

    Merge branch 'gerrit/phoenix' into 'master'
    
    Change-Id: Ib9aab9747180152f456b1e330b209685ad38a402

 .../org/apache/asterix/active/ActiveManager.java   |   1 +
 .../optimizer/rules/PushFieldAccessRule.java       |  13 +-
 .../optimizer/rules/cbo/EnumerateJoinsRule.java    |  14 +-
 .../asterix/optimizer/rules/cbo/JoinNode.java      | 107 ++--
 .../apache/asterix/translator/ClientRequest.java   |   4 +
 .../asterix/translator/IStatementExecutor.java     |  33 +-
 .../apache/asterix/translator/ResultMetadata.java  |  38 +-
 .../apache/asterix/translator/SessionConfig.java   |  21 +-
 .../apache/asterix/api/common/APIFramework.java    |  13 +-
 .../api/http/server/AbstractQueryApiServlet.java   | 195 +++++++
 .../api/http/server/NCQueryResultApiServlet.java   |  61 +++
 .../api/http/server/NCQueryServiceServlet.java     |   7 +-
 .../api/http/server/NCQueryStatusApiServlet.java   |  20 +-
 .../api/http/server/QueryResultApiServlet.java     | 141 +++--
 .../http/server/QueryServiceRequestParameters.java |  14 +-
 .../api/http/server/QueryServiceServlet.java       | 156 +-----
 .../api/http/server/QueryStatusApiServlet.java     | 115 +++-
 .../apache/asterix/api/http/server/ResultUtil.java |  21 +
 .../asterix/api/http/server/StorageApiServlet.java |  11 +-
 .../external/ExternalCredentialsCacheUpdater.java  |   4 +-
 .../message/AbstractInternalRequestMessage.java    |   2 +
 .../app/message/ClientInfoRequestMessage.java      |  67 +++
 .../app/message/ClientInfoResponseMessage.java     |  50 ++
 .../DiscardResultPartitionRequestMessage.java      |  57 ++
 .../app/message/DiscardResultRequestMessage.java   |  62 +++
 .../app/message/DiscardResultResponseMessage.java  |  44 ++
 .../message/ExecuteStatementRequestMessage.java    |  32 +-
 .../app/message/StorageCleanupRequestMessage.java  |   4 +-
 .../apache/asterix/app/nc/NCAppRuntimeContext.java |   4 +-
 .../org/apache/asterix/app/nc/RecoveryManager.java |  34 +-
 .../org/apache/asterix/app/nc/ReplicaManager.java  |  37 +-
 .../nc/task/CloudToLocalStorageCachingTask.java    |  14 +-
 .../asterix/app/nc/task/LocalRecoveryTask.java     |  12 +-
 .../app/nc/task/LocalStorageCleanupTask.java       |   5 +-
 .../asterix/app/nc/task/UpdateNodeStatusTask.java  |   9 +-
 .../app/replication/NcLifecycleCoordinator.java    |  13 +-
 .../message/NCLifecycleTaskReportMessage.java      |  11 +-
 .../message/RegistrationTasksRequestMessage.java   |  12 +-
 .../message/RegistrationTasksResponseMessage.java  |   5 +-
 .../asterix/app/result/JobResultCallback.java      |   3 +
 .../apache/asterix/app/result/ResultHandle.java    |  75 ++-
 .../apache/asterix/app/result/ResultReader.java    |  10 +
 .../app/result/fields/CreatedAtPrinter.java}       |  35 +-
 .../asterix/app/result/fields/MetricsPrinter.java  | 151 ++++--
 .../asterix/app/result/fields/NcResultPrinter.java |  12 +-
 .../app/result/fields/PartitionInfoPrinter.java    |  64 +++
 .../app/result/fields/ResultCountPrinter.java}     |  32 +-
 .../asterix/app/translator/QueryTranslator.java    | 152 ++++--
 .../asterix/hyracks/bootstrap/CCApplication.java   |   9 +
 .../asterix/hyracks/bootstrap/NCApplication.java   |  17 +-
 .../apache/asterix/messaging/CCMessageBroker.java  |   9 +
 .../apache/asterix/utils/AsyncRequestsAPIUtil.java | 151 ++++++
 .../org/apache/asterix/utils/RedactionUtil.java    |   2 +-
 .../asterix/runtime/ClusterStateManagerTest.java   |   7 +-
 .../asterix/test/common/ResultExtractor.java       |   7 +-
 .../apache/asterix/test/common/TestExecutor.java   |  55 +-
 .../dataflow/GlobalVirtualBufferCacheTest.java     | 100 ++++
 .../async-compilation-failed.1.async.sqlpp         |   1 -
 .../async-exhausted-result.1.async.sqlpp           |   1 -
 .../async-exhausted-result.2.pollget.http}         |   6 +-
 .../async-exhausted-result.3.get.http}             |   4 +-
 .../async-exhausted-result.4.delete.http}          |   4 +-
 .../async-exhausted-result.5.get.http}             |   4 +-
 .../async-failed/async-failed.1.async.sqlpp        |   1 -
 .../async-failed/async-failed.2.pollget.http}      |   5 +-
 .../async-json/async-json.1.async.sqlpp            |   1 -
 .../async-json/async-json.2.pollget.http}          |   5 +-
 .../async-json/async-json.3.get.http}              |   6 +-
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 -
 .../async-repeated/async-repeated.2.pollget.http}  |   5 +-
 .../async-repeated/async-repeated.3.get.http}      |   4 +-
 .../async-repeated/async-repeated.4.get.http}      |   4 +-
 .../async-repeated/async-repeated.5.query.sqlpp}   |   2 -
 .../async-running/async-running.1.async.sqlpp      |   1 -
 .../async-running/async-running.2.pollget.http}    |   4 +-
 .../async-running/async-running.3.pollget.http}    |   5 +-
 .../async-running/async-running.4.get.http}        |   4 +-
 .../async-timeout}/async.1.async.sqlpp             |   4 +-
 .../async-timeout/async.2.pollget.http}            |   4 +-
 .../async/async.1.async.sqlpp                      |   1 -
 .../async/async.2.pollget.http}                    |   5 +-
 .../async/async.3.get.http}                        |   4 +-
 .../deferred/deferred.1.deferred.sqlpp}            |   3 +-
 .../deferred/deferred.2.get.http}                  |   4 +-
 .../async-deferred/AsyncDeferredQueries.xml        |  52 ++
 .../async-compilation-failed.1.async.sqlpp         |   1 +
 .../async-exhausted-result.1.async.sqlpp           |   1 +
 .../async-failed/async-failed.1.async.sqlpp        |   1 +
 .../async-json/async-json.1.async.sqlpp            |   1 +
 .../async-repeated/async-repeated.1.async.sqlpp    |   1 +
 .../async-running/async-running.1.async.sqlpp      |   1 +
 .../async-deferred/async/async.1.async.sqlpp       |   1 +
 .../parquet-error-checks.23.ddl.sqlpp              |   1 +
 .../parquet-null1.01.ddl.sqlpp}                    |   3 +
 .../parquet-null1.02.update.sqlpp}                 |  25 +-
 .../parquet-null1.03.ddl.sqlpp}                    |  15 +-
 .../parquet-null1/parquet-null1.04.query.sqlpp}    |   7 +-
 .../parquet-null2.01.ddl.sqlpp}                    |   5 +-
 .../parquet-null2.02.update.sqlpp}                 |  17 +-
 .../parquet-null2.03.ddl.sqlpp}                    |  15 +-
 .../parquet-null2/parquet-null2.04.query.sqlpp}    |   7 +-
 .../parquet-simple/parquet-simple.01.ddl.sqlpp     |   2 +
 .../parquet-simple/parquet-simple.02.update.sqlpp  |   4 +-
 .../queries_sqlpp/flwor/at00/at00.6.async.sqlpp    |   1 +
 .../async-compilation-failed.1.regex               |   2 +
 .../async-exhausted-result.1.ignore                |   0
 .../async-exhausted-result.2.regexjson             |   8 +
 .../async-exhausted-result.3.regexjson             |  17 +
 .../async-exhausted-result.4.ignore                |  17 +
 .../async-failed/async-failed.1.ignore             |   0
 .../async-failed/async-failed.2.regex              |   2 +
 .../async-json/async-json.1.ignore                 |   0
 .../async-json/async-json.2.regexjson              |   8 +
 .../async-json/async-json.3.json                   |   5 +
 .../async-repeated/async-repeated.1.ignore         |   0
 .../async-repeated/async-repeated.2.regexjson      |   8 +
 .../async-repeated/async-repeated.3.regexjson      |  17 +
 .../async-repeated/async-repeated.4.regexjson      |  17 +
 .../async-repeated/async-repeated.5.json           |  10 +
 .../async-running/async-running.1.ignore           |   0
 .../async-running/async-running.2.json             |   3 +
 .../async-running/async-running.3.regexjson        |   8 +
 .../async-running/async-running.4.regexjson        |   8 +
 .../async-timeout/async-timeout.1.ignore           |   0
 .../async-timeout/async-timeout.2.regex            |   3 +
 .../async-deferred-improved/async/async.1.ignore   |   0
 .../async/async.2.regexjson                        |   8 +
 .../async/async.3.regexjson                        |  17 +
 .../deferred/deferred.1.ignore                     |   0
 .../deferred/deferred.2.regexjson                  |  17 +
 .../copy-to/parquet-null1/parquet-null1.04.adm     |   2 +
 .../copy-to/parquet-null2/parquet-null2.04.adm     |   3 +
 .../single-collection-queries.3.plan               |  26 +-
 .../single-collection-queries.4.plan               |  26 +-
 .../single-collection-queries.7.plan               |  30 +-
 .../runtimets/testsuite_external_dataset_s3.xml    |  21 +-
 .../resources/runtimets/testsuite_sqlpp_hdfs.xml   |   1 -
 .../asterix/cloud/AbstractCloudIOManager.java      |  12 +-
 .../asterix/cloud/LocalPartitionBootstrapper.java  |   4 +-
 .../cloud/clients/aws/s3/S3ClientConfig.java       |  22 +-
 .../cloud/clients/aws/s3/S3ParallelDownloader.java |   2 +-
 .../blobstorage/AzBlobStorageClientConfig.java     |   2 +-
 .../blobstorage/AzBlobStorageCloudClient.java      | 101 ++--
 .../cloud/lazy/accessor/InitialCloudAccessor.java  |   5 +-
 .../lazy/accessor/ReplaceableCloudAccessor.java    |   5 +-
 .../lazy/accessor/SelectiveCloudAccessor.java      |   4 +-
 .../apache/asterix/cloud/util/CloudFileUtil.java   |   1 -
 .../writer/AzureExternalFileWriterFactory.java     |   2 +-
 .../cloud/writer/S3ExternalFileWriterFactory.java  |   4 +-
 .../org/apache/asterix/cloud/s3/LSMS3Test.java     |   2 +-
 .../lsm/flush/FlushColumnTupleWriter.java          |  14 +-
 .../apache/asterix/common/api/IRequestTracker.java |   8 +
 .../common/cloud/IPartitionBootstrapper.java       |   4 +-
 .../common/cluster/IClusterStateManager.java       |   3 +-
 .../cluster/StorageComputePartitionsMap.java       |   7 +-
 .../asterix/common/config/CloudProperties.java     |  75 +--
 .../asterix/common/config/MetadataProperties.java  |   4 +-
 .../asterix/common/config/PropertiesAccessor.java  |   7 +-
 .../common/context/DatasetLifecycleManager.java    |  19 +-
 .../common/context/GlobalVirtualBufferCache.java   |   6 +-
 .../common/messaging/api/ICCMessageBroker.java     |   8 +
 .../asterix/common/storage/IReplicaManager.java    |  10 +-
 .../common/transactions/IRecoveryManager.java      |   8 +-
 .../org/apache/asterix/common/utils/Datasets.java  | 105 ++++
 .../asterix/common/utils/IntSortedBitSet.java      | 234 +++++++++
 .../org/apache/asterix/common/utils/IntUtil.java   | 119 +++++
 .../org/apache/asterix/common/utils/MathUtil.java  |  53 ++
 .../apache/asterix/common/utils/Partitions.java    | 240 +++++++++
 .../asterix/common/utils/ShortSortedBitSet.java    | 235 +++++++++
 .../org/apache/asterix/common/utils/ShortUtil.java | 122 +++++
 .../apache/asterix/test/utils/MathUtilTest.java    |  79 +++
 .../asterix/test/utils/ShortSortedBitSetTest.java  | 108 ++++
 .../apache/asterix/external/api/IDataParser.java   |  13 +-
 .../asterix/external/api/IRecordConverter.java     |   8 +-
 .../dataflow/FeedRecordDataFlowController.java     |   2 +
 .../external/input/HDFSDataSourceFactory.java      |   3 +-
 .../input/record/reader/aws/AwsS3InputStream.java  |   9 +-
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  |   2 +-
 .../aws/parquet/AwsS3ParquetReaderFactory.java     |   2 +-
 .../asterix/external/util/ExternalDataUtils.java   |  11 +-
 .../asterix/external/util/aws/AwsConstants.java    |  42 ++
 .../apache/asterix/external/util/aws/AwsUtils.java | 286 ++++++++++
 .../asterix/external/util/aws/s3/S3AuthUtils.java  | 577 ---------------------
 .../asterix/external/util/aws/s3/S3Constants.java  |  16 -
 .../asterix/external/util/aws/s3/S3Utils.java      | 313 ++++++++++-
 .../util/azure/blob_storage/AzureConstants.java    |   2 +-
 .../util/azure/blob_storage/AzureUtils.java        |  21 +-
 .../writer/HDFSExternalFileWriterFactory.java      |   2 +-
 .../writer/printer/parquet/ParquetSchemaTree.java  |  21 +-
 .../printer/parquet/SchemaCheckerLazyVisitor.java  |   6 +-
 .../input/record/reader/awss3/AwsS3Test.java       |   4 +-
 .../metadata/declared/MetadataProvider.java        |   8 +
 .../apache/asterix/metadata/utils/DatasetUtil.java |   6 +-
 .../replication/logging/RemoteLogsNotifier.java    |   7 +-
 .../replication/sync/ReplicaSynchronizer.java      |   4 +-
 .../asterix/runtime/utils/ClusterStateManager.java |  12 +-
 .../asterix/runtime/utils/RequestTracker.java      |  18 +
 .../PersistentLocalResourceRepository.java         |  11 +-
 asterixdb/pom.xml                                  |  16 +-
 ...ReplacementsAwareJavaSerializationProvider.java |  82 +++
 .../api/result/IResultPartitionManager.java        |   3 +-
 .../org/apache/hyracks/api/result/IResultSet.java  |   3 +
 .../hyracks/api/result/IResultSetReader.java       |   2 +
 .../hyracks/api/result/ResultDirectoryRecord.java  |  23 +-
 .../apache/hyracks/api/result/ResultJobRecord.java |  22 +-
 .../hyracks/api/util/JavaSerializationUtils.java   |  39 +-
 .../client/result/PartitionResultSetReader.java    | 112 ++++
 .../apache/hyracks/client/result/ResultSet.java    |  13 +
 .../hyracks/client/result/ResultSetReader.java     |  49 +-
 .../hyracks/control/cc/ClusterControllerIPCI.java  |   4 +-
 .../control/cc/result/IResultDirectoryService.java |   6 +-
 .../control/cc/result/ResultDirectoryService.java  |  24 +-
 .../control/cc/work/GetResultMetadataWork.java     |   3 -
 .../work/RegisterResultPartitionLocationWork.java  |   7 +-
 .../ReportResultPartitionWriteCompletionWork.java  |   7 +-
 .../control/common/base/IClusterController.java    |   5 +-
 .../hyracks/control/common/ipc/CCNCFunctions.java  |  19 +-
 .../common/ipc/ClusterControllerRemoteProxy.java   |  10 +-
 .../control/nc/result/ResultPartitionManager.java  |   9 +-
 .../control/nc/result/ResultPartitionWriter.java   |   9 +-
 .../hyracks/control/nc/result/ResultState.java     |   9 +-
 .../std/util/ByteArrayAccessibleOutputStream.java  |  18 +
 .../hyracks/hyracks-dataflow-std/pom.xml           |   1 -
 .../std/result/ResultWriterOperatorDescriptor.java |   2 +
 .../apache/hyracks/http/server/utils/HttpUtil.java |   6 +-
 .../dataflow/IndexDropOperatorNodePushable.java    |   2 +-
 .../storage/am/lsm/common/impls/LSMHarness.java    |  31 +-
 .../apache/hyracks/util/ThrowingIOFunction.java    |  20 +-
 228 files changed, 4886 insertions(+), 1602 deletions(-)

diff --cc asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 230d6fb662,5e4706c8a8..4da12b4f98
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@@ -5518,8 -5529,9 +5534,9 @@@ public class QueryTranslator extends Ab
                  afterCompile();
                  MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
                  stats.setCompileTime(System.nanoTime() - compileStart);
+                 resultMetadata.setCompileTime(stats.getCompileTime());
                  bActiveTxn = false;
 -                return query.isExplain() || isCompileOnly() ? null : jobSpec;
 +                return query.isExplain() || query.isAdvise() || isCompileOnly() ? null : jobSpec;
              } catch (Exception e) {
                  LOGGER.log(Level.INFO, e.getMessage(), e);
                  if (bActiveTxn) {


(asterixdb) 22/25: [ASTERIXDB-3650][COMP] Ignore sample indexes when checking for secondary indexes

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit c8542b55f04eb69ab4d6286f7a7147341d9cfeb8
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Tue Oct 21 14:40:49 2025 -0700

    [ASTERIXDB-3650][COMP] Ignore sample indexes when checking for secondary indexes
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    Currently DatasetUtil.hasSecondaries, does not
    account for sample indexes, which will lead to lookup
    in LSMPrimaryUpsertOperatorNodePushable.
    
    Ext-ref: MB-68717
    Change-Id: I65ca9c74192d14bbaecf8a306902e4cecf1337d2
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20410
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
 .../apache/asterix/optimizer/rules/PushFieldAccessRule.java | 13 +------------
 .../single-collection-queries.7.plan                        |  8 ++++----
 .../apache/asterix/metadata/declared/MetadataProvider.java  |  8 ++++++++
 .../java/org/apache/asterix/metadata/utils/DatasetUtil.java |  6 +++---
 4 files changed, 16 insertions(+), 19 deletions(-)

diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
index 31f2511667..7356bd82d9 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
@@ -33,7 +33,6 @@ import org.apache.asterix.metadata.declared.DataSource;
 import org.apache.asterix.metadata.declared.DataSourceId;
 import org.apache.asterix.metadata.declared.MetadataProvider;
 import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.Index;
 import org.apache.asterix.metadata.utils.DatasetUtil;
 import org.apache.asterix.om.base.AInt32;
 import org.apache.asterix.om.base.AString;
@@ -152,17 +151,7 @@ public class PushFieldAccessRule implements IAlgebraicRewriteRule {
             }
         }
 
-        List<Index> datasetIndexes =
-                mp.getDatasetIndexes(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName());
-        boolean hasSecondaryIndex = false;
-        for (Index index : datasetIndexes) {
-            if (index.isSecondaryIndex()) {
-                hasSecondaryIndex = true;
-                break;
-            }
-        }
-
-        return hasSecondaryIndex;
+        return mp.hasSecondaryIndexes(dataset);
     }
 
     private boolean tryingToPushThroughSelectionWithSameDataSource(AssignOperator access, AbstractLogicalOperator op2) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
index cd177f77b9..5754b6d01e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/cardinality-estimation/single-collection-queries/single-collection-queries.7.plan
@@ -26,10 +26,10 @@ distribute result [$$51] [cardinality: 136.0, doc-size: 0.0, op-cost: 0.0, total
               -- EXTERNAL_GROUP_BY[$$52]  |PARTITIONED|
                 exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                 -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
-                  select (and(gt($$53, 1), gt($$54, 4))) project: [$$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
-                  -- STREAM_SELECT  |PARTITIONED|
-                    assign [$$52] <- [$$l.getField(1)] project: [$$53, $$54, $$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
-                    -- ASSIGN  |PARTITIONED|
+                  assign [$$52] <- [$$l.getField(1)] project: [$$52] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
+                  -- ASSIGN  |PARTITIONED|
+                    select (and(gt($$53, 1), gt($$54, 4))) project: [$$l] [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
+                    -- STREAM_SELECT  |PARTITIONED|
                       exchange [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
                       -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
                         unnest-map [$$53, $$54, $$l] <- index-search("LineItem", 0, "Default", "tpch", "LineItem", false, false, 2, $$60, $$61, 0, true, true, false) [cardinality: 1321.89, doc-size: 0.0, op-cost: 0.0, total-cost: 6005.0]
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index b0291375b2..8ab2f2aa52 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -1918,6 +1918,14 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String>
                 .filter(idx -> idx.isSecondaryIndex() && !idx.isSampleIndex()).collect(Collectors.toList());
     }
 
+    /**
+     * Returns true if the dataset has any (non-samples) secondary index.
+     */
+    public boolean hasSecondaryIndexes(Dataset ds) throws AlgebricksException {
+        return getDatasetIndexes(ds.getDatabaseName(), ds.getDataverseName(), ds.getDatasetName()).stream()
+                .anyMatch(idx -> idx.isSecondaryIndex() && !idx.isSampleIndex());
+    }
+
     public LockList getLocks() {
         return locks;
     }
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 55f42599ca..c39395b094 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -484,9 +484,9 @@ public class DatasetUtil {
             primaryKeyFields[i] = i;
             pkFields[i] = fieldPermutation[i];
         }
-        boolean hasSecondaries = metadataProvider
-                .getDatasetIndexes(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName())
-                .size() > 1;
+
+        boolean hasSecondaries = metadataProvider.hasSecondaryIndexes(dataset);
+
         IStorageComponentProvider storageComponentProvider = metadataProvider.getStorageComponentProvider();
         IModificationOperationCallbackFactory modificationCallbackFactory = dataset.getModificationCallbackFactory(
                 storageComponentProvider, primaryIndex, IndexOperation.UPSERT, primaryKeyFields);


(asterixdb) 24/25: [NO ISSUE][EXT]: change azure cloud client to accept both clients

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 39ac77d88a3a24246057079de6d42f2742680e4b
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Oct 22 20:25:36 2025 +0300

    [NO ISSUE][EXT]: change azure cloud client to accept both clients
    
    Ext-ref: MB-69069
    Change-Id: I1ee625e30cccf44aeebfa4d5ae5aa99968942ac5
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20511
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Hussain Towaileb <hu...@gmail.com>
    Reviewed-by: Michael Blow <mb...@apache.org>
---
 .../azure/blobstorage/AzBlobStorageCloudClient.java |  6 +++---
 .../writer/AzureExternalFileWriterFactory.java      |  2 +-
 .../util/azure/blob_storage/AzureUtils.java         | 21 ++++++++++++++++++---
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
index 0bd16a867b..1cd0556789 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/clients/azure/blobstorage/AzBlobStorageCloudClient.java
@@ -100,13 +100,13 @@ public class AzBlobStorageCloudClient implements ICloudClient {
     private static final Logger LOGGER = LogManager.getLogger();
 
     public AzBlobStorageCloudClient(AzBlobStorageClientConfig config, ICloudGuardian guardian) {
-        this(config, buildClient(config), guardian);
+        this(config, buildClient(config), buildAsyncClient(config), guardian);
     }
 
     public AzBlobStorageCloudClient(AzBlobStorageClientConfig config, BlobServiceClient blobServiceClient,
-            ICloudGuardian guardian) {
+            BlobServiceAsyncClient asyncBlobServiceClient, ICloudGuardian guardian) {
         this.blobContainerClient = blobServiceClient.getBlobContainerClient(config.getBucket());
-        this.blobContainerAsyncClient = buildAsyncClient(config).getBlobContainerAsyncClient(config.getBucket());
+        this.blobContainerAsyncClient = asyncBlobServiceClient.getBlobContainerAsyncClient(config.getBucket());
         this.config = config;
         this.guardian = guardian;
         long profilerInterval = config.getProfilerLogInterval();
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
index fd71ffc63c..d9eb16dd0f 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AzureExternalFileWriterFactory.java
@@ -68,7 +68,7 @@ public final class AzureExternalFileWriterFactory extends AbstractCloudExternalF
     ICloudClient createCloudClient(IApplicationContext appCtx) throws CompilationException {
         AzBlobStorageClientConfig config = AzBlobStorageClientConfig.of(configuration, writeBufferSize);
         return new AzBlobStorageCloudClient(config, AzureUtils.buildAzureBlobClient(appCtx, configuration),
-                ICloudGuardian.NoOpCloudGuardian.INSTANCE);
+                AzureUtils.buildAzureBlobAsyncClient(appCtx, configuration), ICloudGuardian.NoOpCloudGuardian.INSTANCE);
     }
 
     @Override
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
index ac407f4dc8..4d2a2dd116 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/azure/blob_storage/AzureUtils.java
@@ -77,6 +77,7 @@ import com.azure.identity.ClientCertificateCredentialBuilder;
 import com.azure.identity.ClientSecretCredentialBuilder;
 import com.azure.identity.ManagedIdentityCredentialBuilder;
 import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobServiceAsyncClient;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.blob.BlobServiceClientBuilder;
 import com.azure.storage.blob.models.BlobItem;
@@ -94,14 +95,24 @@ public class AzureUtils {
         throw new AssertionError("do not instantiate");
     }
 
+    public static BlobServiceClient buildAzureBlobClient(IApplicationContext appCtx, Map<String, String> configuration)
+            throws CompilationException {
+        return buildAzureBlobClient(BlobServiceClient.class, appCtx, configuration);
+    }
+
+    public static BlobServiceAsyncClient buildAzureBlobAsyncClient(IApplicationContext appCtx,
+            Map<String, String> configuration) throws CompilationException {
+        return buildAzureBlobClient(BlobServiceAsyncClient.class, appCtx, configuration);
+    }
+
     /**
      * Builds the Azure storage account using the provided configuration
      *
      * @param configuration properties
      * @return client
      */
-    public static BlobServiceClient buildAzureBlobClient(IApplicationContext appCtx, Map<String, String> configuration)
-            throws CompilationException {
+    private static <T> T buildAzureBlobClient(Class<T> type, IApplicationContext appCtx,
+            Map<String, String> configuration) throws CompilationException {
         String managedIdentityId = configuration.get(MANAGED_IDENTITY_ID_FIELD_NAME);
         String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
         String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
@@ -247,7 +258,11 @@ public class AzureUtils {
         }
 
         try {
-            return builder.buildClient();
+            if (type == BlobServiceClient.class) {
+                return type.cast(builder.buildClient());
+            } else {
+                return type.cast(builder.buildAsyncClient());
+            }
         } catch (Exception ex) {
             throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
         }


(asterixdb) 20/25: [ASTERIXDB-3648][COMP] Fix incorrect replacement of Left Outer Joins with Unnest in CBO

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 8c18a52aa2c0dc0bf95bee8b62b0b5e4d8ab55c8
Author: preetham0202 <pr...@couchbase.com>
AuthorDate: Wed Sep 24 15:33:17 2025 +0530

    [ASTERIXDB-3648][COMP] Fix incorrect replacement of Left Outer Joins with Unnest in CBO
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Ext-ref: MB-68691
    
    Change-Id: I3f1d4d093cf1bd23f5db75fce569fd174e8cedb4
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/20426
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Preetham Poluparthi <pr...@apache.org>
    Reviewed-by: <mu...@couchbase.com>
    Tested-by: Preetham Poluparthi <pr...@apache.org>
---
 .../asterix/optimizer/rules/cbo/EnumerateJoinsRule.java    | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
index 8406520070..b9b35c2760 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/cbo/EnumerateJoinsRule.java
@@ -308,7 +308,7 @@ public class EnumerateJoinsRule implements IAlgebraicRewriteRule {
                 modifyUnnestInfo = new ArrayList<>();
                 collectUnnestModificationInfo(null, root, cheapestPlanNode);
                 for (int k = 0; k < modifyUnnestInfo.size(); k++) {
-                    modifyTree(null, root, k);
+                    modifyTree(null, 0, root, k);
                     if (newRootAfterUnnest != null) {
                         root = newRootAfterUnnest;
                     }
@@ -384,7 +384,7 @@ public class EnumerateJoinsRule implements IAlgebraicRewriteRule {
         return false;
     }
 
-    private void modifyTree(ILogicalOperator parent, ILogicalOperator op, int k) {
+    private void modifyTree(ILogicalOperator parent, int index, ILogicalOperator op, int k) {
         if (modifyUnnestInfo.get(k).second == op) { // found the one to get rid off; this should be an OJ
             int size = modifyUnnestInfo.get(k).third.size();
             UnnestOperator uOp = (UnnestOperator) modifyUnnestInfo.get(k).third.get(size - 1); // UnnestOp is always at the end
@@ -394,20 +394,22 @@ public class EnumerateJoinsRule implements IAlgebraicRewriteRule {
                 ILogicalOperator q = newRootAfterUnnest;
                 if (modifyUnnestInfo.get(k).third.size() > 1) {
                     for (ILogicalOperator p : modifyUnnestInfo.get(k).third) {
-                        q.getInputs().get(0).setValue(p);
+                        q.getInputs().get(index).setValue(p);
                         q = p;
                     }
                 }
             } else {
                 ILogicalOperator q = parent;
                 for (ILogicalOperator p : modifyUnnestInfo.get(k).third) {
-                    q.getInputs().get(0).setValue(p);
+                    q.getInputs().get(index).setValue(p);
                     q = p;
                 }
             }
         }
-        for (Mutable<ILogicalOperator> input : op.getInputs()) {
-            modifyTree(op, input.getValue(), k);
+        int size = op.getInputs().size();
+        for (int i = 0; i < size; i++) {
+            Mutable<ILogicalOperator> input = op.getInputs().get(i);
+            modifyTree(op, i, input.getValue(), k);
         }
     }