You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by mb...@apache.org on 2024/11/25 19:43:23 UTC

(asterixdb) branch master updated (d21a38bf55 -> dec9f981e4)

This is an automated email from the ASF dual-hosted git repository.

mblow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git


    from d21a38bf55 [ASTERIXDB-3514][EXT]: Assume role only when temporary credentials expire
     new 6249ff58f3 [ASTERIXDB-3503][EXT] Add column filter for Delta Reader.
     new 49929014b3 [ASTERIXDB-3503][EXT] Improve logic of distributing files to partitions
     new 994f901c8d [ASTERIXDB-3503][EXT] Tests for Field pushdown and delta all type
     new f11758e0d8 [NO ISSUE][HYR][STO,MISC] LSM enhancements, misc utilities
     new 096385e346 [NO ISSUE][HYR][STO] Add pre-exit hook to IFrameOperationCallback
     new 33db60a5ae Merge branch 'gerrit/neo' into 'gerrit/trinity'
     new 1b975b5599 Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
     new 9f75957c6b Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish
     new e9772ce511 [NO ISSUE]: Support Reading credentials from String
     new 5ddcc8f276 [ASTERIXDB-3503][EXT] Fixing Internal Error issue when Delta table does not exists.
     new 4dde600f6c [NO ISSUE][COMP] Exponential recursion in OperatorManipulationUtil.substituteVarRec method
     new 52b8df6313 Merge branch 'gerrit/neo' into 'gerrit/trinity'
     new bdaf4e97e4 Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
     new 7026faf391 Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish
     new dec9f981e4 Merge branch 'gerrit/goldfish' into 'master'

The 15 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 .../rules/PushAggregateIntoNestedSubplanRule.java  |   6 +-
 .../optimizer/rules/PushGroupByThroughProduct.java |   4 +-
 .../external_dataset/ExternalDatasetTestUtils.java |   4 +
 .../deltalake/DeltaAllTypeGenerator.java           | 268 +++++++++++++++++++++
 .../deltalake-all-type.00.ddl.sqlpp                |  52 ++--
 .../deltalake-all-type.01.query.sqlpp              |  30 +--
 .../deltalake-all-type.02.query.sqlpp              |  30 +--
 .../deltalake-field-access-pushdown.00.ddl.sqlpp   |  53 ++--
 .../deltalake-field-access-pushdown.01.query.sqlpp |  30 +--
 .../deltalake-field-access-pushdown.02.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.03.query.sqlpp |  32 +--
 .../deltalake-field-access-pushdown.04.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.05.query.sqlpp |  32 +--
 .../deltalake-field-access-pushdown.06.query.sqlpp |  34 +--
 .../deltalake-field-access-pushdown.07.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.08.query.sqlpp |  34 +--
 .../deltalake-field-access-pushdown.09.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.10.query.sqlpp |  34 +--
 .../deltalake-table-not-exists.00.ddl.sqlpp        |  37 +--
 .../deltalake-all-type/deltalake-all-type.01.adm   |   5 +
 .../deltalake-all-type/deltalake-all-type.02.adm   |   5 +
 .../deltalake-field-access-pushdown.01.adm         |   5 +
 .../deltalake-field-access-pushdown.02.plan        |  22 ++
 .../deltalake-field-access-pushdown.03.adm         |   5 +
 .../deltalake-field-access-pushdown.04.plan        |  22 ++
 .../deltalake-field-access-pushdown.05.adm         |   5 +
 .../deltalake-field-access-pushdown.06.plan        |  50 ++++
 .../deltalake-field-access-pushdown.07.adm         |   5 +
 .../deltalake-field-access-pushdown.08.plan        |  50 ++++
 .../deltalake-field-access-pushdown.09.adm         |   5 +
 .../deltalake-field-access-pushdown.10.adm         |  46 ++++
 .../runtimets/testsuite_external_dataset_s3.xml    |  19 ++
 .../NoOpFrameOperationCallbackFactory.java         |   5 +
 .../aws/delta/AsterixDeltaRuntimeException.java    |  34 +--
 .../aws/delta/AsterixTypeToDeltaTypeVisitor.java   | 145 +++++++++++
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  | 150 +++++++++---
 .../reader/aws/delta/DeltaFileRecordReader.java    |  10 +-
 .../aws/delta/converter/DeltaConverterContext.java |  10 +-
 .../asterix/external/parser/DeltaDataParser.java   |  13 +-
 .../asterix/external/util/ExternalDataUtils.java   |  34 ++-
 .../asterix/external/util/aws/s3/S3AuthUtils.java  |   4 +
 .../external/util/google/gcs/GCSConstants.java     |  12 +
 .../asterix/external/util/google/gcs/GCSUtils.java |  35 ++-
 .../LSMPrimaryInsertOperatorNodePushable.java      |  10 +-
 .../LSMPrimaryUpsertOperatorNodePushable.java      |  16 +-
 .../runtime/operators/StandardBatchController.java |  45 ++++
 .../algebra/util/OperatorManipulationUtil.java     |  19 +-
 .../rewriter/rules/AbstractDecorrelationRule.java  |   4 +-
 .../subplan/IntroduceGroupByForSubplanRule.java    |   7 +-
 .../apache/hyracks/api/comm/IFrameAppender.java    |   2 +-
 .../hyracks/api/util/HyracksThrowingAction.java}   |  30 +--
 .../org/apache/hyracks/api/util/InvokeUtil.java    |  36 +++
 .../common/io/MessagingFrameTupleAppender.java     |   9 -
 .../hyracks/dataflow/common/utils/TaskUtil.java    |  14 ++
 ...perationCallback.java => IBatchController.java} |  29 +--
 .../am/lsm/common/api/IFrameOperationCallback.java |  14 +-
 .../storage/am/lsm/common/api/ILSMHarness.java     |   9 +-
 .../storage/am/lsm/common/impls/LSMHarness.java    |  31 ++-
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |   6 +-
 59 files changed, 1232 insertions(+), 557 deletions(-)
 create mode 100644 asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaAllTypeGenerator.java
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.00.ddl.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.01.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.02.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.00.ddl.sqlpp (50%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.query.sqlpp (52%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.query.sqlpp (51%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-table-not-exists/deltalake-table-not-exists.00.ddl.sqlpp (52%)
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.01.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.02.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.plan
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.plan
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.plan
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.plan
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.adm
 create mode 100644 asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.adm
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixDeltaRuntimeException.java (56%)
 create mode 100644 asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
 create mode 100644 asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
 copy hyracks-fullstack/hyracks/{hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java => hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingAction.java} (55%)
 copy hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/{IFrameOperationCallback.java => IBatchController.java} (59%)


(asterixdb) 07/15: [ASTERIXDB-3503][EXT] Tests for Field pushdown and delta all type

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 994f901c8d3bdeda0eed72aff463544eea171e1a
Author: ayush.tripathi <ay...@couchbase.com>
AuthorDate: Tue Nov 19 16:13:01 2024 +0530

    [ASTERIXDB-3503][EXT] Tests for Field pushdown and delta all type
    
    - user model changes: yes
    - storage format changes: no
    - interface changes: no
    
    Details:
    - Tests to check optional flags of Deltalake like "decimal-to-double", "date-to-long" and "timestamp-to-long".
    - Testing plans for column filter.
    
    Ext-ref: MB-63840
    
    Change-Id: I8b9dc57a792f88a28467a791a8d0b678ade7d82d
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19091
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Tested-by: Murtadha Hubail <mh...@apache.org>
    Integration-Tests: Murtadha Hubail <mh...@apache.org>
---
 .../external_dataset/ExternalDatasetTestUtils.java |   4 +
 .../deltalake/DeltaAllTypeGenerator.java           | 268 +++++++++++++++++++++
 .../deltalake-all-type.00.ddl.sqlpp                |  50 ++++
 .../deltalake-all-type.01.query.sqlpp              |  22 ++
 .../deltalake-all-type.02.query.sqlpp              |  22 ++
 .../deltalake-field-access-pushdown.00.ddl.sqlpp   |  51 ++++
 .../deltalake-field-access-pushdown.01.query.sqlpp |  28 +++
 .../deltalake-field-access-pushdown.02.query.sqlpp |  27 +++
 .../deltalake-field-access-pushdown.03.query.sqlpp |  26 ++
 .../deltalake-field-access-pushdown.04.query.sqlpp |  27 +++
 .../deltalake-field-access-pushdown.05.query.sqlpp |  28 +++
 .../deltalake-field-access-pushdown.06.query.sqlpp |  28 +++
 .../deltalake-field-access-pushdown.07.query.sqlpp |  27 +++
 .../deltalake-field-access-pushdown.08.query.sqlpp |  28 +++
 .../deltalake-field-access-pushdown.09.query.sqlpp |  27 +++
 .../deltalake-field-access-pushdown.10.query.sqlpp |  28 +++
 .../deltalake-all-type/deltalake-all-type.01.adm   |   5 +
 .../deltalake-all-type/deltalake-all-type.02.adm   |   5 +
 .../deltalake-field-access-pushdown.01.adm         |   5 +
 .../deltalake-field-access-pushdown.02.plan        |  22 ++
 .../deltalake-field-access-pushdown.03.adm         |   5 +
 .../deltalake-field-access-pushdown.04.plan        |  22 ++
 .../deltalake-field-access-pushdown.05.adm         |   5 +
 .../deltalake-field-access-pushdown.06.plan        |  50 ++++
 .../deltalake-field-access-pushdown.07.adm         |   5 +
 .../deltalake-field-access-pushdown.08.plan        |  50 ++++
 .../deltalake-field-access-pushdown.09.adm         |   5 +
 .../deltalake-field-access-pushdown.10.adm         |  46 ++++
 .../runtimets/testsuite_external_dataset_s3.xml    |  12 +
 29 files changed, 928 insertions(+)

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index db1bf5039b..07789b764f 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -37,6 +37,7 @@ import java.nio.file.Paths;
 import java.util.Collection;
 
 import org.apache.asterix.test.external_dataset.avro.AvroFileConverterUtil;
+import org.apache.asterix.test.external_dataset.deltalake.DeltaAllTypeGenerator;
 import org.apache.asterix.test.external_dataset.deltalake.DeltaTableGenerator;
 import org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil;
 import org.apache.asterix.testframework.context.TestCaseContext;
@@ -122,6 +123,7 @@ public class ExternalDatasetTestUtils {
         // cleaning directory
         BinaryFileConverterUtil.cleanBinaryDirectory(basePath, DELTA_GEN_BASEDIR);
         DeltaTableGenerator.prepareDeltaTableContainer(new Configuration());
+        DeltaAllTypeGenerator.createTableInsertData(new Configuration());
     }
 
     /**
@@ -434,6 +436,8 @@ public class ExternalDatasetTestUtils {
         loadDeltaDirectory(generatedDataBasePath, "/modified_delta_table/_delta_log", JSON_FILTER, "delta-data/");
         loadDeltaDirectory(generatedDataBasePath, "/multiple_file_delta_table", PARQUET_FILTER, "delta-data/");
         loadDeltaDirectory(generatedDataBasePath, "/multiple_file_delta_table/_delta_log", JSON_FILTER, "delta-data/");
+        loadDeltaDirectory(generatedDataBasePath, "/delta_all_type/_delta_log", JSON_FILTER, "delta-data/");
+        loadDeltaDirectory(generatedDataBasePath, "/delta_all_type", PARQUET_FILTER, "delta-data/");
     }
 
     private static void loadDeltaDirectory(String dataBasePath, String rootPath, FilenameFilter filter,
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaAllTypeGenerator.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaAllTypeGenerator.java
new file mode 100644
index 0000000000..5c302a5f90
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/deltalake/DeltaAllTypeGenerator.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.external_dataset.deltalake;
+
+import static io.delta.kernel.internal.util.Utils.toCloseableIterator;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.DataWriteContext;
+import io.delta.kernel.Operation;
+import io.delta.kernel.Table;
+import io.delta.kernel.Transaction;
+import io.delta.kernel.TransactionBuilder;
+import io.delta.kernel.data.ColumnVector;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.defaults.internal.data.DefaultColumnarBatch;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.DataType;
+import io.delta.kernel.types.DateType;
+import io.delta.kernel.types.DecimalType;
+import io.delta.kernel.types.DoubleType;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.types.TimestampType;
+import io.delta.kernel.utils.CloseableIterable;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.DataFileStatus;
+
+public class DeltaAllTypeGenerator {
+    public static final DecimalType decimal_t = new DecimalType(10, 5);
+    protected static final StructType exampleTableSchema = new StructType().add("integer_type", IntegerType.INTEGER)
+            .add("string_type", StringType.STRING).add("decimal_type", decimal_t).add("double_type", DoubleType.DOUBLE)
+            .add("timestamp_type", TimestampType.TIMESTAMP).add("date_type", DateType.DATE);
+    public static final String DELTA_ALL_TYPE_TABLE_PATH =
+            "target" + File.separatorChar + "generated_delta_files" + File.separatorChar + "delta_all_type";
+
+    public static void createTableInsertData(Configuration conf) throws IOException {
+        Engine engine = DefaultEngine.create(conf);
+        Table table = Table.forPath(engine, DELTA_ALL_TYPE_TABLE_PATH);
+        TransactionBuilder txnBuilder = table.createTransactionBuilder(engine, "Examples", Operation.CREATE_TABLE);
+        txnBuilder = txnBuilder.withSchema(engine, exampleTableSchema);
+        Transaction txn = txnBuilder.build(engine);
+        Row txnState = txn.getTransactionState(engine);
+        ColumnVector[] vectors = new ColumnVector[exampleTableSchema.length()];
+        vectors[0] = intVector(Arrays.asList(123, 124, 125, 126, 127));
+        vectors[1] = stringVector(
+                Arrays.asList("FirstPerson", "SecondPerson", "ThirdPerson", "FourthPerson", "FifthPerson"));
+        vectors[2] = decimalVector(Arrays.asList(new BigDecimal("1.25432"), new BigDecimal("2666.223"),
+                new BigDecimal("1245.2421"), new BigDecimal("23731.2"), new BigDecimal("80911.222456")));
+        vectors[3] = doubleVector(Arrays.asList(100.34d, 200.055d, 300.02d, 400.21014d, 500.219d));
+        vectors[4] = timestampVector(
+                Arrays.asList(1732010400000L, 1732010400330L, 1732010400450L, 1732010403000L, 1732010401200L));
+        vectors[5] = dateVector(Arrays.asList(127, 23, 11, 456, 23));
+        ColumnarBatch batch = new DefaultColumnarBatch(5, exampleTableSchema, vectors);
+        FilteredColumnarBatch f1 = new FilteredColumnarBatch(batch, Optional.empty());
+        CloseableIterator<FilteredColumnarBatch> data = toCloseableIterator(Arrays.asList(f1).iterator());
+        CloseableIterator<FilteredColumnarBatch> physicalData =
+                Transaction.transformLogicalData(engine, txnState, data, Collections.emptyMap());
+        DataWriteContext writeContext = Transaction.getWriteContext(engine, txnState, Collections.emptyMap());
+        CloseableIterator<DataFileStatus> dataFiles = engine.getParquetHandler().writeParquetFiles(
+                writeContext.getTargetDirectory(), physicalData, writeContext.getStatisticsColumns());
+        CloseableIterator<Row> dataActions =
+                Transaction.generateAppendActions(engine, txnState, dataFiles, writeContext);
+        CloseableIterable<Row> dataActionsIterable = CloseableIterable.inMemoryIterable(dataActions);
+        txn.commit(engine, dataActionsIterable);
+
+    }
+
+    static ColumnVector stringVector(List<String> data) {
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return StringType.STRING;
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return data.get(rowId) == null;
+            }
+
+            @Override
+            public String getString(int rowId) {
+                return data.get(rowId);
+            }
+        };
+    }
+
+    static ColumnVector intVector(List<Integer> data) {
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return IntegerType.INTEGER;
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return false;
+            }
+
+            @Override
+            public int getInt(int rowId) {
+                return data.get(rowId);
+            }
+        };
+    }
+
+    static ColumnVector doubleVector(List<Double> data) {
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return DoubleType.DOUBLE;
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return data.get(rowId) == null;
+            }
+
+            @Override
+            public double getDouble(int rowId) {
+                return data.get(rowId);
+            }
+        };
+    }
+
+    static ColumnVector decimalVector(List<BigDecimal> data) {
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return decimal_t; // Use the specific DecimalType passed (scale and precision)
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return data.get(rowId) == null;
+            }
+
+            @Override
+            public BigDecimal getDecimal(int rowId) {
+                // Return the BigDecimal value directly as Delta Kernel works natively with BigDecimal for decimals
+                return data.get(rowId);
+            }
+        };
+    }
+
+    static ColumnVector timestampVector(List<Long> data) { // Assuming timestamp values are stored as microseconds since epoch
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return TimestampType.TIMESTAMP;
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return data.get(rowId) == null;
+            }
+
+            @Override
+            public long getLong(int rowId) {
+                // Delta Lake often uses microseconds since epoch for timestamps
+                return data.get(rowId);
+            }
+        };
+    }
+
+    static ColumnVector dateVector(List<Integer> data) { // Assuming date values are stored as days since epoch
+        return new ColumnVector() {
+            @Override
+            public DataType getDataType() {
+                return DateType.DATE;
+            }
+
+            @Override
+            public int getSize() {
+                return data.size();
+            }
+
+            @Override
+            public void close() {
+            }
+
+            @Override
+            public boolean isNullAt(int rowId) {
+                return data.get(rowId) == null;
+            }
+
+            @Override
+            public int getInt(int rowId) {
+                // Delta Lake often uses days since epoch for dates
+                return data.get(rowId);
+            }
+        };
+    }
+
+}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.00.ddl.sqlpp
new file mode 100644
index 0000000000..cdae363774
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.00.ddl.sqlpp
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+
+CREATE TYPE DeltalakeTableType as {
+ };
+
+CREATE EXTERNAL COLLECTION DeltalakeDataset1(DeltalakeTableType) USING %adapter%
+ (
+   %template%,
+   ("container"="playground"),
+   ("definition"="delta-data/delta_all_type"),
+   ("decimal-to-double" = "true"),
+   ("timestamp-to-long" = "false"),
+   ("date-to-int" = "false"),
+   ("timezone" =  "PST"),
+   ("table-format" = "delta")
+ );
+
+CREATE EXTERNAL COLLECTION DeltalakeDataset2(DeltalakeTableType) USING %adapter%
+  (
+    %template%,
+    ("container"="playground"),
+    ("definition"="delta-data/delta_all_type"),
+    ("decimal-to-double" = "true"),
+    ("timestamp-to-long" = "true"),
+    ("date-to-int" = "true"),
+    ("table-format" = "delta")
+  );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.01.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.01.query.sqlpp
new file mode 100644
index 0000000000..90f80ab159
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.01.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT element ds FROM DeltalakeDataset1 as ds;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.02.query.sqlpp
new file mode 100644
index 0000000000..b75203b184
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-all-type/deltalake-all-type.02.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT element ds FROM DeltalakeDataset2 as ds;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.00.ddl.sqlpp
new file mode 100644
index 0000000000..4987403348
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.00.ddl.sqlpp
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL COLLECTION DeltalakeDataset1(DeltalakeTableType) USING %adapter%
+ (
+   %template%,
+   ("container"="playground"),
+   ("definition"="delta-data/delta_all_type"),
+   ("decimal-to-double" = "true"),
+   ("timestamp-to-long"="false"),
+   ("date-to-int"="false"),
+   ("timezone" = "PST"),
+   ("table-format" = "delta")
+ );
+
+  CREATE EXTERNAL COLLECTION DeltalakeDataset2(DeltalakeTableType) USING %adapter%
+  (
+    %template%,
+    ("container"="playground"),
+    ("definition"="delta-data/delta_all_type"),
+    ("decimal-to-double" = "true"),
+    ("timestamp-to-long"="false"),
+    ("date-to-int"="false"),
+    ("timezone" = "PST"),
+    ("table-format" = "delta")
+  );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.query.sqlpp
new file mode 100644
index 0000000000..41805993ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+USE test;
+
+SET `compiler.external.field.pushdown` "false";
+
+
+SELECT VALUE d
+FROM DeltalakeDataset1 d
+ORDER BY d.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.query.sqlpp
new file mode 100644
index 0000000000..d3009591a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "false";
+
+Explain
+SELECT VALUE d
+FROM DeltalakeDataset1 d
+ORDER BY d.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.query.sqlpp
new file mode 100644
index 0000000000..073171cc55
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "true";
+
+SELECT VALUE d
+FROM DeltalakeDataset1 d
+ORDER BY d.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.query.sqlpp
new file mode 100644
index 0000000000..d3009591a3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "false";
+
+Explain
+SELECT VALUE d
+FROM DeltalakeDataset1 d
+ORDER BY d.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.query.sqlpp
new file mode 100644
index 0000000000..fa60442b46
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "false";
+
+
+SELECT d1.integer_type, d2.string_type, d2.timestamp_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.query.sqlpp
new file mode 100644
index 0000000000..e1eceddfb3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "false";
+
+EXPLAIN
+SELECT d1.integer_type, d2.string_type, d2.timestamp_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.query.sqlpp
new file mode 100644
index 0000000000..1e440bfb93
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "true";
+
+SELECT d1.integer_type, d2.string_type, d2.timestamp_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.query.sqlpp
new file mode 100644
index 0000000000..786618af96
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "true";
+
+Explain
+SELECT d1.integer_type, d2.string_type, d2.timestamp_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.query.sqlpp
new file mode 100644
index 0000000000..d2c91531fb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "true";
+
+SELECT d1.integer_type, d1.date_type, d2.decimal_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.query.sqlpp
new file mode 100644
index 0000000000..bb4701a976
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SET `compiler.external.field.pushdown` "true";
+
+Explain
+SELECT d1.integer_type, d1.date_type, d2.decimal_type
+FROM DeltalakeDataset1 d1, DeltalakeDataset2 d2
+WHERE d1.integer_type = d2.integer_type
+ORDER BY d2.integer_type;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.01.adm
new file mode 100644
index 0000000000..e5f99a3486
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.01.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 123, "string_type": "FirstPerson", "decimal_type": 1.25432, "double_type": 100.34, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-05-08") }
+{ "integer_type": 124, "string_type": "SecondPerson", "decimal_type": 2666.223, "double_type": 200.055, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-24") }
+{ "integer_type": 125, "string_type": "ThirdPerson", "decimal_type": 1245.2421, "double_type": 300.02, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-12") }
+{ "integer_type": 126, "string_type": "FourthPerson", "decimal_type": 23731.2, "double_type": 400.21014, "timestamp_type": datetime("1970-01-20T17:06:50.403"), "date_type": date("1971-04-02") }
+{ "integer_type": 127, "string_type": "FifthPerson", "decimal_type": 80911.22245, "double_type": 500.219, "timestamp_type": datetime("1970-01-20T17:06:50.401"), "date_type": date("1970-01-24") }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.02.adm
new file mode 100644
index 0000000000..9cc21d6f59
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-all-type/deltalake-all-type.02.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 123, "string_type": "FirstPerson", "decimal_type": 1.25432, "double_type": 100.34, "timestamp_type": 1732010400, "date_type": 127 }
+{ "integer_type": 124, "string_type": "SecondPerson", "decimal_type": 2666.223, "double_type": 200.055, "timestamp_type": 1732010400, "date_type": 23 }
+{ "integer_type": 125, "string_type": "ThirdPerson", "decimal_type": 1245.2421, "double_type": 300.02, "timestamp_type": 1732010400, "date_type": 11 }
+{ "integer_type": 126, "string_type": "FourthPerson", "decimal_type": 23731.2, "double_type": 400.21014, "timestamp_type": 1732010403, "date_type": 456 }
+{ "integer_type": 127, "string_type": "FifthPerson", "decimal_type": 80911.22245, "double_type": 500.219, "timestamp_type": 1732010401, "date_type": 23 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.adm
new file mode 100644
index 0000000000..e5f99a3486
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.01.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 123, "string_type": "FirstPerson", "decimal_type": 1.25432, "double_type": 100.34, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-05-08") }
+{ "integer_type": 124, "string_type": "SecondPerson", "decimal_type": 2666.223, "double_type": 200.055, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-24") }
+{ "integer_type": 125, "string_type": "ThirdPerson", "decimal_type": 1245.2421, "double_type": 300.02, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-12") }
+{ "integer_type": 126, "string_type": "FourthPerson", "decimal_type": 23731.2, "double_type": 400.21014, "timestamp_type": datetime("1970-01-20T17:06:50.403"), "date_type": date("1971-04-02") }
+{ "integer_type": 127, "string_type": "FifthPerson", "decimal_type": 80911.22245, "double_type": 500.219, "timestamp_type": datetime("1970-01-20T17:06:50.401"), "date_type": date("1970-01-24") }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.plan
new file mode 100644
index 0000000000..6f01ee7583
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.02.plan
@@ -0,0 +1,22 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT  |PARTITIONED|
+  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+    project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+    -- STREAM_PROJECT  |PARTITIONED|
+      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+      -- SORT_MERGE_EXCHANGE [$$14(ASC) ]  |PARTITIONED|
+        order (ASC, $$14) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+        -- STABLE_SORT [$$14(ASC)]  |PARTITIONED|
+          exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+          -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+            assign [$$14] <- [$$d.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+            -- ASSIGN  |PARTITIONED|
+              exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+              -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                data-scan []<-[$$d] <- test.DeltalakeDataset1 [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                -- DATASOURCE_SCAN  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                    empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.adm
new file mode 100644
index 0000000000..e5f99a3486
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.03.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 123, "string_type": "FirstPerson", "decimal_type": 1.25432, "double_type": 100.34, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-05-08") }
+{ "integer_type": 124, "string_type": "SecondPerson", "decimal_type": 2666.223, "double_type": 200.055, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-24") }
+{ "integer_type": 125, "string_type": "ThirdPerson", "decimal_type": 1245.2421, "double_type": 300.02, "timestamp_type": datetime("1970-01-20T17:06:50.400"), "date_type": date("1970-01-12") }
+{ "integer_type": 126, "string_type": "FourthPerson", "decimal_type": 23731.2, "double_type": 400.21014, "timestamp_type": datetime("1970-01-20T17:06:50.403"), "date_type": date("1971-04-02") }
+{ "integer_type": 127, "string_type": "FifthPerson", "decimal_type": 80911.22245, "double_type": 500.219, "timestamp_type": datetime("1970-01-20T17:06:50.401"), "date_type": date("1970-01-24") }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.plan
new file mode 100644
index 0000000000..6f01ee7583
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.04.plan
@@ -0,0 +1,22 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT  |PARTITIONED|
+  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+    project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+    -- STREAM_PROJECT  |PARTITIONED|
+      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+      -- SORT_MERGE_EXCHANGE [$$14(ASC) ]  |PARTITIONED|
+        order (ASC, $$14) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+        -- STABLE_SORT [$$14(ASC)]  |PARTITIONED|
+          exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+          -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+            assign [$$14] <- [$$d.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+            -- ASSIGN  |PARTITIONED|
+              exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+              -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                data-scan []<-[$$d] <- test.DeltalakeDataset1 [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                -- DATASOURCE_SCAN  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                    empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.adm
new file mode 100644
index 0000000000..1f480e4c12
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.05.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 125, "string_type": "ThirdPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
+{ "integer_type": 126, "string_type": "FourthPerson", "timestamp_type": datetime("1970-01-20T17:06:50.403") }
+{ "integer_type": 127, "string_type": "FifthPerson", "timestamp_type": datetime("1970-01-20T17:06:50.401") }
+{ "integer_type": 124, "string_type": "SecondPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
+{ "integer_type": 123, "string_type": "FirstPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.plan
new file mode 100644
index 0000000000..848bc0456a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.06.plan
@@ -0,0 +1,50 @@
+distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT  |PARTITIONED|
+  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+    project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+    -- STREAM_PROJECT  |PARTITIONED|
+      assign [$$32] <- [{"integer_type": $$34, "string_type": $$37, "timestamp_type": $$38}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+      -- ASSIGN  |PARTITIONED|
+        project ([$$34, $$37, $$38]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+        -- STREAM_PROJECT  |PARTITIONED|
+          exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+          -- SORT_MERGE_EXCHANGE [$$36(ASC) ]  |PARTITIONED|
+            order (ASC, $$36) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+            -- STABLE_SORT [$$36(ASC)]  |PARTITIONED|
+              exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+              -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                project ([$$34, $$37, $$38, $$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                -- STREAM_PROJECT  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                    join (eq($$34, $$35)) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- HYBRID_HASH_JOIN [$$34][$$35]  |PARTITIONED|
+                      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- HASH_PARTITION_EXCHANGE [$$34]  |PARTITIONED|
+                        project ([$$34]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- STREAM_PROJECT  |PARTITIONED|
+                          assign [$$34] <- [$$d1.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- ASSIGN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              data-scan []<-[$$d1] <- test.DeltalakeDataset1 [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- DATASOURCE_SCAN  |PARTITIONED|
+                                exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                                  empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                  -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
+                      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- HASH_PARTITION_EXCHANGE [$$35]  |PARTITIONED|
+                        project ([$$37, $$38, $$36, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- STREAM_PROJECT  |PARTITIONED|
+                          assign [$$38, $$37, $$36, $$35] <- [$$d2.getField("timestamp_type"), $$d2.getField("string_type"), $$d2.getField("id"), $$d2.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- ASSIGN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              data-scan []<-[$$d2] <- test.DeltalakeDataset2 [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- DATASOURCE_SCAN  |PARTITIONED|
+                                exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                                  empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                  -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.adm
new file mode 100644
index 0000000000..1f480e4c12
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.07.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 125, "string_type": "ThirdPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
+{ "integer_type": 126, "string_type": "FourthPerson", "timestamp_type": datetime("1970-01-20T17:06:50.403") }
+{ "integer_type": 127, "string_type": "FifthPerson", "timestamp_type": datetime("1970-01-20T17:06:50.401") }
+{ "integer_type": 124, "string_type": "SecondPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
+{ "integer_type": 123, "string_type": "FirstPerson", "timestamp_type": datetime("1970-01-20T17:06:50.400") }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.plan
new file mode 100644
index 0000000000..34671ddc4d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.08.plan
@@ -0,0 +1,50 @@
+distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT  |PARTITIONED|
+  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+    project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+    -- STREAM_PROJECT  |PARTITIONED|
+      assign [$$32] <- [{"integer_type": $$34, "string_type": $$37, "timestamp_type": $$38}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+      -- ASSIGN  |PARTITIONED|
+        project ([$$34, $$37, $$38]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+        -- STREAM_PROJECT  |PARTITIONED|
+          exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+          -- SORT_MERGE_EXCHANGE [$$36(ASC) ]  |PARTITIONED|
+            order (ASC, $$36) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+            -- STABLE_SORT [$$36(ASC)]  |PARTITIONED|
+              exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+              -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                project ([$$34, $$37, $$38, $$36]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                -- STREAM_PROJECT  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                    join (eq($$34, $$35)) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- HYBRID_HASH_JOIN [$$34][$$35]  |PARTITIONED|
+                      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- HASH_PARTITION_EXCHANGE [$$34]  |PARTITIONED|
+                        project ([$$34]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- STREAM_PROJECT  |PARTITIONED|
+                          assign [$$34] <- [$$d1.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- ASSIGN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              data-scan []<-[$$d1] <- test.DeltalakeDataset1 project ({integer_type:any}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- DATASOURCE_SCAN  |PARTITIONED|
+                                exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                                  empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                  -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
+                      exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- HASH_PARTITION_EXCHANGE [$$35]  |PARTITIONED|
+                        project ([$$37, $$38, $$36, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- STREAM_PROJECT  |PARTITIONED|
+                          assign [$$38, $$37, $$36, $$35] <- [$$d2.getField("timestamp_type"), $$d2.getField("string_type"), $$d2.getField("id"), $$d2.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- ASSIGN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              data-scan []<-[$$d2] <- test.DeltalakeDataset2 project ({string_type:any,timestamp_type:any,id:any,integer_type:any}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- DATASOURCE_SCAN  |PARTITIONED|
+                                exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                                  empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                                  -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.adm
new file mode 100644
index 0000000000..5e1edce4c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.09.adm
@@ -0,0 +1,5 @@
+{ "integer_type": 123, "date_type": date("1970-05-08"), "decimal_type": 1.25432 }
+{ "integer_type": 124, "date_type": date("1970-01-24"), "decimal_type": 2666.223 }
+{ "integer_type": 125, "date_type": date("1970-01-12"), "decimal_type": 1245.2421 }
+{ "integer_type": 126, "date_type": date("1971-04-02"), "decimal_type": 23731.2 }
+{ "integer_type": 127, "date_type": date("1970-01-24"), "decimal_type": 80911.22245 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.adm
new file mode 100644
index 0000000000..f706ba28dd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/deltalake-field-access-pushdown/deltalake-field-access-pushdown.10.adm
@@ -0,0 +1,46 @@
+distribute result [$$32] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT  |PARTITIONED|
+  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+  -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+    project ([$$32]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+    -- STREAM_PROJECT  |PARTITIONED|
+      assign [$$32] <- [{"integer_type": $$34, "date_type": $$37, "decimal_type": $$38}] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+      -- ASSIGN  |PARTITIONED|
+        project ([$$34, $$37, $$38]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+        -- STREAM_PROJECT  |PARTITIONED|
+          exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+          -- SORT_MERGE_EXCHANGE [$$35(ASC) ]  |PARTITIONED|
+            order (ASC, $$35) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+            -- STABLE_SORT [$$35(ASC)]  |PARTITIONED|
+              exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+              -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                join (eq($$34, $$35)) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                -- HYBRID_HASH_JOIN [$$34][$$35]  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- HASH_PARTITION_EXCHANGE [$$34]  |PARTITIONED|
+                    project ([$$34, $$37]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- STREAM_PROJECT  |PARTITIONED|
+                      assign [$$37, $$34] <- [$$d1.getField("date_type"), $$d1.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- ASSIGN  |PARTITIONED|
+                        exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                          data-scan []<-[$$d1] <- test.DeltalakeDataset1 project ({date_type:any,integer_type:any}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- DATASOURCE_SCAN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
+                  exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                  -- HASH_PARTITION_EXCHANGE [$$35]  |PARTITIONED|
+                    project ([$$38, $$35]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                    -- STREAM_PROJECT  |PARTITIONED|
+                      assign [$$38, $$35] <- [$$d2.getField("decimal_type"), $$d2.getField("integer_type")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                      -- ASSIGN  |PARTITIONED|
+                        exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                        -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                          data-scan []<-[$$d2] <- test.DeltalakeDataset2 project ({decimal_type:any,integer_type:any}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                          -- DATASOURCE_SCAN  |PARTITIONED|
+                            exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                            -- ONE_TO_ONE_EXCHANGE  |PARTITIONED|
+                              empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+                              -- EMPTY_TUPLE_SOURCE  |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 2c7af49eb0..d56c1a46ec 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -320,6 +320,18 @@
         <output-dir compare="Text">common/deltalake-modified-data</output-dir>
       </compilation-unit>
     </test-case>
+    <test-case FilePath="external-dataset">
+      <compilation-unit name="common/deltalake-all-type">
+        <placeholder name="adapter" value="S3" />
+        <output-dir compare="Text">common/deltalake-all-type</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="external-dataset">
+      <compilation-unit name="common/deltalake-field-access-pushdown">
+        <placeholder name="adapter" value="S3" />
+        <output-dir compare="Text">common/deltalake-field-access-pushdown</output-dir>
+      </compilation-unit>
+    </test-case>
     <test-case FilePath="external-dataset">
       <compilation-unit name="common/deltalake-invalid-file-format">
         <placeholder name="adapter" value="S3" />


(asterixdb) 04/15: [NO ISSUE][HYR][STO] Add pre-exit hook to IFrameOperationCallback

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 096385e34610ed4cbb238124f66282cdaa3a7e49
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Tue Nov 19 11:00:01 2024 -0500

    [NO ISSUE][HYR][STO] Add pre-exit hook to IFrameOperationCallback
    
    Ext-ref: MB-64229
    Change-Id: I911884a0f4f6d66d750e452b6b8049ad67d0b00a
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19092
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../dataflow/NoOpFrameOperationCallbackFactory.java  |  5 +++++
 .../LSMPrimaryUpsertOperatorNodePushable.java        |  5 +++++
 .../runtime/operators/StandardBatchController.java   |  9 ++++++---
 .../storage/am/lsm/common/api/IBatchController.java  |  6 ++++--
 .../am/lsm/common/api/IFrameOperationCallback.java   | 14 ++++++++++++--
 .../storage/am/lsm/common/api/ILSMHarness.java       |  3 ++-
 .../storage/am/lsm/common/impls/LSMHarness.java      | 20 ++++++++++++++++----
 7 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
index 915f3b34e2..3ab86c0cfa 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/NoOpFrameOperationCallbackFactory.java
@@ -46,6 +46,11 @@ public class NoOpFrameOperationCallbackFactory implements IFrameOperationCallbac
             // No Op
         }
 
+        @Override
+        public void beforeExit(boolean success) throws HyracksDataException {
+            // No Op
+        }
+
         @Override
         public void close() throws IOException {
             // No Op
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 643c6abef1..d2dc3cffb5 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -292,6 +292,11 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe
                     callback.frameCompleted();
                 }
 
+                @Override
+                public void beforeExit(boolean success) throws HyracksDataException {
+                    callback.beforeExit(success);
+                }
+
                 @Override
                 public void close() throws IOException {
                     callback.close();
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
index f9f758b64f..40465c6c5f 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
@@ -20,6 +20,7 @@ package org.apache.asterix.runtime.operators;
 
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
+import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
 import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
@@ -31,12 +32,14 @@ class StandardBatchController implements IBatchController {
     }
 
     @Override
-    public void batchEnter(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException {
+    public void batchEnter(ILSMIndexOperationContext ctx, ILSMHarness lsmHarness, IFrameOperationCallback callback)
+            throws HyracksDataException {
         lsmHarness.enter(ctx, LSMOperationType.MODIFICATION);
     }
 
     @Override
-    public void batchExit(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException {
-        lsmHarness.exit(ctx, LSMOperationType.MODIFICATION);
+    public void batchExit(ILSMIndexOperationContext ctx, ILSMHarness lsmHarness, IFrameOperationCallback callback,
+            boolean batchSuccessful) throws HyracksDataException {
+        lsmHarness.exit(ctx, callback, batchSuccessful, LSMOperationType.MODIFICATION);
     }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java
index 879a8d223b..e0615897ec 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java
@@ -23,7 +23,9 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 public interface IBatchController {
     String KEY_BATCH_CONTROLLER = "BATCH_CONTROLLER";
 
-    void batchEnter(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException;
+    void batchEnter(ILSMIndexOperationContext ctx, ILSMHarness lsmHarness, IFrameOperationCallback callback)
+            throws HyracksDataException;
 
-    void batchExit(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException;
+    void batchExit(ILSMIndexOperationContext ctx, ILSMHarness lsmHarness, IFrameOperationCallback callback,
+            boolean batchSuccessful) throws HyracksDataException;
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
index 1f89af2f09..2fbc0c5fac 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IFrameOperationCallback.java
@@ -28,14 +28,24 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
 public interface IFrameOperationCallback extends Closeable {
     /**
      * Called once processing the frame is done before calling nextFrame on the next IFrameWriter in
-     * the pipeline
+     * the pipeline. In the event this frame completion will also exit the component, this will be
+     * called prior to {@link #beforeExit(boolean)}.
      *
      * @throws HyracksDataException
      */
     void frameCompleted() throws HyracksDataException;
 
     /**
-     * Called when the task has failed.
+     * Called just prior to exiting the component on batch completion: not all batches may result
+     * in a component exit, depending on the decision of the {@link IBatchController}.
+     *
+     * @throws HyracksDataException
+     */
+    void beforeExit(boolean success) throws HyracksDataException;
+
+    /**
+     * Called when the batch processing, {@link #frameCompleted()} or {@link #beforeExit(boolean)}
+     * invocation has failed.
      *
      * @param th
      */
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 721d809c64..68de45ac08 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -228,7 +228,8 @@ public interface ILSMHarness {
 
     void enter(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException;
 
-    void exit(ILSMIndexOperationContext ctx, LSMOperationType op) throws HyracksDataException;
+    void exit(ILSMIndexOperationContext ctx, IFrameOperationCallback callback, boolean success, LSMOperationType op)
+            throws HyracksDataException;
 
     /**
      * Rollback components that match the passed predicate
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 6e206869b1..017e767721 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -698,8 +698,18 @@ public class LSMHarness implements ILSMHarness {
     }
 
     @Override
-    public void exit(ILSMIndexOperationContext ctx, LSMOperationType op) throws HyracksDataException {
-        getAndExitComponentsAndComplete(ctx, op);
+    public void exit(ILSMIndexOperationContext ctx, IFrameOperationCallback callback, boolean success,
+            LSMOperationType op) throws HyracksDataException {
+        try {
+            callback.beforeExit(success);
+        } catch (Throwable th) {
+            // TODO(mblow): we don't distinguish between the three distinct phases we can encounter
+            //              failures in the callback API- we might need this eventually
+            callback.fail(th);
+            throw th;
+        } finally {
+            getAndExitComponentsAndComplete(ctx, op);
+        }
     }
 
     private void getAndExitComponentsAndComplete(ILSMIndexOperationContext ctx, LSMOperationType op)
@@ -717,10 +727,12 @@ public class LSMHarness implements ILSMHarness {
             IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController)
             throws HyracksDataException {
         processor.start();
-        batchController.batchEnter(this, ctx);
+        batchController.batchEnter(ctx, this, frameOpCallback);
+        boolean success = false;
         try {
             try {
                 processFrame(accessor, tuple, processor);
+                success = true;
                 frameOpCallback.frameCompleted();
             } catch (Throwable th) {
                 processor.fail(th);
@@ -732,7 +744,7 @@ public class LSMHarness implements ILSMHarness {
             LOGGER.warn("Failed to process frame", e);
             throw e;
         } finally {
-            batchController.batchExit(this, ctx);
+            batchController.batchExit(ctx, this, frameOpCallback, success);
             ctx.logPerformanceCounters(accessor.getTupleCount());
         }
     }


(asterixdb) 09/15: Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 9f75957c6b278e0ce3756b634a46a1cd3cfe8da6
Merge: 994f901c8d 1b975b5599
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Thu Nov 21 20:20:55 2024 +0000

    Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish

 .../NoOpFrameOperationCallbackFactory.java         |  5 +++
 .../LSMPrimaryInsertOperatorNodePushable.java      | 10 ++++-
 .../LSMPrimaryUpsertOperatorNodePushable.java      | 16 +++++++-
 .../runtime/operators/StandardBatchController.java | 45 ++++++++++++++++++++++
 .../apache/hyracks/api/comm/IFrameAppender.java    |  2 +-
 .../hyracks/api/util/HyracksThrowingAction.java}   | 30 ++-------------
 .../org/apache/hyracks/api/util/InvokeUtil.java    | 36 +++++++++++++++++
 .../common/io/MessagingFrameTupleAppender.java     |  9 -----
 .../hyracks/dataflow/common/utils/TaskUtil.java    | 14 +++++++
 ...perationCallback.java => IBatchController.java} | 29 +++-----------
 .../am/lsm/common/api/IFrameOperationCallback.java | 14 ++++++-
 .../storage/am/lsm/common/api/ILSMHarness.java     |  9 ++++-
 .../storage/am/lsm/common/impls/LSMHarness.java    | 31 +++++++++++----
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |  6 ++-
 14 files changed, 180 insertions(+), 76 deletions(-)


(asterixdb) 08/15: Merge branch 'gerrit/trinity' into 'gerrit/goldfish'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 1b975b5599a0091bec3ee272194233912810c4e4
Merge: 49929014b3 33db60a5ae
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Nov 20 16:04:08 2024 -0500

    Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
    
    Ext-ref: MB-64229,MB-63741
    Change-Id: Ia47a6630f71e75b181afe8cdddd5dcfdbbda4a05

 .../NoOpFrameOperationCallbackFactory.java         |  5 +++
 .../LSMPrimaryInsertOperatorNodePushable.java      | 10 ++++-
 .../LSMPrimaryUpsertOperatorNodePushable.java      | 16 +++++++-
 .../runtime/operators/StandardBatchController.java | 45 ++++++++++++++++++++++
 .../apache/hyracks/api/comm/IFrameAppender.java    |  2 +-
 .../hyracks/api/util/HyracksThrowingAction.java}   | 30 ++-------------
 .../org/apache/hyracks/api/util/InvokeUtil.java    | 36 +++++++++++++++++
 .../common/io/MessagingFrameTupleAppender.java     |  9 -----
 .../hyracks/dataflow/common/utils/TaskUtil.java    | 14 +++++++
 ...perationCallback.java => IBatchController.java} | 29 +++-----------
 .../am/lsm/common/api/IFrameOperationCallback.java | 14 ++++++-
 .../storage/am/lsm/common/api/ILSMHarness.java     |  9 ++++-
 .../storage/am/lsm/common/impls/LSMHarness.java    | 31 +++++++++++----
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |  6 ++-
 14 files changed, 180 insertions(+), 76 deletions(-)

diff --cc asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
index 072a9b6ae7,5c879944fd..9b8b103d55
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
@@@ -18,15 -18,13 +18,17 @@@
   */
  package org.apache.asterix.runtime.operators;
  
+ import static org.apache.hyracks.storage.am.lsm.common.api.IBatchController.KEY_BATCH_CONTROLLER;
+ 
  import java.nio.ByteBuffer;
 +import java.util.HashMap;
 +import java.util.Map;
  
  import org.apache.asterix.common.api.INcApplicationContext;
 +import org.apache.asterix.common.context.PrimaryIndexOperationTracker;
  import org.apache.asterix.common.dataflow.LSMIndexUtil;
  import org.apache.asterix.common.dataflow.NoOpFrameOperationCallbackFactory;
 +import org.apache.asterix.common.messaging.AtomicJobPreparedMessage;
  import org.apache.asterix.common.transactions.ILogMarkerCallback;
  import org.apache.asterix.common.transactions.PrimaryIndexLogMarkerCallback;
  import org.apache.asterix.transaction.management.opcallbacks.LockThenSearchOperationCallback;
@@@ -57,51 -53,38 +59,53 @@@ import org.apache.hyracks.storage.am.co
  import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
  import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
  import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
 +import org.apache.hyracks.storage.am.common.util.ResourceReleaseUtils;
+ import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
  import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
  import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor;
 +import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponentId;
 +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
 +import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
  import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMIndexInsertUpdateDeleteOperatorNodePushable;
  import org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndex;
 +import org.apache.hyracks.storage.am.lsm.common.impls.FlushOperation;
  import org.apache.hyracks.storage.am.lsm.common.impls.LSMTreeIndexAccessor;
  import org.apache.hyracks.storage.common.IIndex;
  import org.apache.hyracks.storage.common.IIndexAccessParameters;
 +import org.apache.hyracks.storage.common.IIndexAccessor;
  import org.apache.hyracks.storage.common.IIndexCursor;
 +import org.apache.hyracks.storage.common.ISearchOperationCallback;
  import org.apache.hyracks.storage.common.MultiComparator;
  
 +import it.unimi.dsi.fastutil.ints.Int2ObjectMap;
 +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap;
 +import it.unimi.dsi.fastutil.ints.IntArrayList;
 +import it.unimi.dsi.fastutil.ints.IntList;
 +import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
 +import it.unimi.dsi.fastutil.ints.IntSet;
 +
  public class LSMPrimaryInsertOperatorNodePushable extends LSMIndexInsertUpdateDeleteOperatorNodePushable {
  
 -    private final IIndexDataflowHelper keyIndexHelper;
 +    private final boolean[] keyIndexHelpersOpen;
 +    private final IIndexDataflowHelper[] keyIndexHelpers;
      private MultiComparator keySearchCmp;
      private RangePredicate searchPred;
 -    private IIndexCursor cursor;
 -    private LockThenSearchOperationCallback searchCallback;
 +    private final IIndexCursor[] cursors;
 +    private final ISearchOperationCallback[] searchCallbacks;
      private final ISearchOperationCallbackFactory searchCallbackFactory;
 -    private final IFrameTupleProcessor processor;
 -    private LSMTreeIndexAccessor lsmAccessor;
 -    private LSMTreeIndexAccessor lsmAccessorForKeyIndex;
 -    private LSMTreeIndexAccessor lsmAccessorForUniqunessCheck;
 +    private final IFrameTupleProcessor[] processors;
 +    private final LSMTreeIndexAccessor[] lsmAccessorForKeyIndexes;
 +    private final LSMTreeIndexAccessor[] lsmAccessorForUniqunessChecks;
  
 -    private final IFrameOperationCallback frameOpCallback;
 +    private final IFrameOperationCallback[] frameOpCallbacks;
-     private boolean flushedPartialTuples;
 +    private final PermutingFrameTupleReference keyTuple;
 +    private final Int2ObjectMap<IntSet> partition2TuplesMap = new Int2ObjectOpenHashMap<>();
 +    private final IntSet processedTuples = new IntOpenHashSet();
 +    private final IntSet flushedTuples = new IntOpenHashSet();
 +    private final SourceLocation sourceLoc;
+     private boolean flushedPartialTuples;
 -    private int currentTupleIdx;
 -    private int lastFlushedTupleIdx;
+     private IBatchController batchController;
  
 -    private final PermutingFrameTupleReference keyTuple;
 -
      public LSMPrimaryInsertOperatorNodePushable(IHyracksTaskContext ctx, int partition,
              IIndexDataflowHelperFactory indexHelperFactory, IIndexDataflowHelperFactory keyIndexHelperFactory,
              int[] fieldPermutation, RecordDescriptor inputRecDesc,
@@@ -228,62 -182,51 +232,63 @@@
          flushedPartialTuples = false;
          accessor = new FrameTupleAccessor(inputRecDesc);
          writeBuffer = new VSizeFrame(ctx);
 -        indexHelper.open();
 -        index = indexHelper.getIndexInstance();
 -        IIndex indexForUniquessCheck;
 -        if (keyIndexHelper != null) {
 -            keyIndexHelper.open();
 -            indexForUniquessCheck = keyIndexHelper.getIndexInstance();
 -        } else {
 -            indexForUniquessCheck = index;
 -        }
          try {
 -            if (ctx.getSharedObject() != null) {
 -                PrimaryIndexLogMarkerCallback callback = new PrimaryIndexLogMarkerCallback((AbstractLSMIndex) index);
 -                TaskUtil.put(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
 -            }
 -            frameOpCallback.open();
 +            INcApplicationContext appCtx =
 +                    (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
              writer.open();
 +            writerOpen = true;
 +            for (int i = 0; i < partitions.length; i++) {
 +                IIndexDataflowHelper indexHelper = indexHelpers[i];
 +                indexHelpersOpen[i] = true;
 +                indexHelper.open();
 +                indexes[i] = indexHelper.getIndexInstance();
 +                IIndex index = indexes[i];
 +                if (((ILSMIndex) indexes[i]).isAtomic()) {
 +                    ((PrimaryIndexOperationTracker) ((ILSMIndex) indexes[i]).getOperationTracker()).clear();
 +                }
 +                IIndexDataflowHelper keyIndexHelper = keyIndexHelpers[i];
 +                IIndex indexForUniquessCheck;
 +                if (keyIndexHelper != null) {
 +                    keyIndexHelpersOpen[i] = true;
 +                    keyIndexHelper.open();
 +                    indexForUniquessCheck = keyIndexHelper.getIndexInstance();
 +                } else {
 +                    indexForUniquessCheck = index;
 +                }
 +                if (ctx.getSharedObject() != null && i == 0) {
 +                    PrimaryIndexLogMarkerCallback callback =
 +                            new PrimaryIndexLogMarkerCallback((AbstractLSMIndex) indexes[0]);
 +                    TaskUtil.put(ILogMarkerCallback.KEY_MARKER_CALLBACK, callback, ctx);
 +                }
 +                modCallbacks[i] =
 +                        modOpCallbackFactory.createModificationOperationCallback(indexHelper.getResource(), ctx, this);
 +                searchCallbacks[i] = searchCallbackFactory
 +                        .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, this);
 +                IIndexAccessParameters iap = new IndexAccessParameters(modCallbacks[i], NoOpOperationCallback.INSTANCE);
 +                indexAccessors[i] = index.createAccessor(iap);
 +                if (keyIndexHelper != null) {
 +                    lsmAccessorForKeyIndexes[i] = (LSMTreeIndexAccessor) indexForUniquessCheck.createAccessor(iap);
 +                }
 +                frameOpCallbacks[i] = NoOpFrameOperationCallbackFactory.INSTANCE.createFrameOperationCallback(ctx,
 +                        (ILSMIndexAccessor) indexAccessors[i]);
 +                frameOpCallbacks[i].open();
 +                IIndexAccessParameters iapForUniquenessCheck =
 +                        new IndexAccessParameters(NoOpOperationCallback.INSTANCE, searchCallbacks[i]);
 +                lsmAccessorForUniqunessChecks[i] =
 +                        (LSMTreeIndexAccessor) indexForUniquessCheck.createAccessor(iapForUniquenessCheck);
 +                setAtomicOpContextIfAtomic(indexForUniquessCheck, lsmAccessorForUniqunessChecks[i]);
 +
 +                cursors[i] = lsmAccessorForUniqunessChecks[i].createSearchCursor(false);
 +                LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) index,
 +                        appCtx.getTransactionSubsystem().getLogManager());
 +            }
 +            createTupleProcessors(sourceLoc);
              keySearchCmp =
 -                    BTreeUtils.getSearchMultiComparator(((ITreeIndex) index).getComparatorFactories(), frameTuple);
 +                    BTreeUtils.getSearchMultiComparator(((ITreeIndex) indexes[0]).getComparatorFactories(), frameTuple);
              searchPred = new RangePredicate(frameTuple, frameTuple, true, true, keySearchCmp, keySearchCmp, null, null);
              appender = new FrameTupleAppender(new VSizeFrame(ctx), true);
 -            modCallback =
 -                    modOpCallbackFactory.createModificationOperationCallback(indexHelper.getResource(), ctx, this);
 -            searchCallback = (LockThenSearchOperationCallback) searchCallbackFactory
 -                    .createSearchOperationCallback(indexHelper.getResource().getId(), ctx, this);
 -            IIndexAccessParameters iap = new IndexAccessParameters(modCallback, NoOpOperationCallback.INSTANCE);
 -            indexAccessor = index.createAccessor(iap);
 -            lsmAccessor = (LSMTreeIndexAccessor) indexAccessor;
 -            if (keyIndexHelper != null) {
 -                lsmAccessorForKeyIndex = (LSMTreeIndexAccessor) indexForUniquessCheck.createAccessor(iap);
 -            }
 -
 -            IIndexAccessParameters iapForUniquenessCheck =
 -                    new IndexAccessParameters(NoOpOperationCallback.INSTANCE, searchCallback);
 -            lsmAccessorForUniqunessCheck =
 -                    (LSMTreeIndexAccessor) indexForUniquessCheck.createAccessor(iapForUniquenessCheck);
 -
 -            cursor = lsmAccessorForUniqunessCheck.createSearchCursor(false);
              frameTuple = new FrameTupleReference();
 -            INcApplicationContext appCtx =
 -                    (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
 -            LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) index,
 -                    appCtx.getTransactionSubsystem().getLogManager());
+             batchController = TaskUtil.getOrDefault(KEY_BATCH_CONTROLLER, ctx, StandardBatchController.INSTANCE);
          } catch (Throwable e) { // NOSONAR: Re-thrown
              throw HyracksDataException.create(e);
          }
@@@ -292,21 -235,7 +297,22 @@@
      @Override
      public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
          accessor.reset(buffer);
 -        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController);
 +        partition2TuplesMap.clear();
 +        int itemCount = accessor.getTupleCount();
 +        for (int i = 0; i < itemCount; i++) {
 +            int storagePartition = tuplePartitioner.partition(accessor, i);
 +            int pIdx = storagePartitionId2Index.get(storagePartition);
 +            IntSet tupleIndexes = partition2TuplesMap.computeIfAbsent(pIdx, k -> new IntOpenHashSet());
 +            tupleIndexes.add(i);
 +        }
 +        for (Int2ObjectMap.Entry<IntSet> p2tuplesMapEntry : partition2TuplesMap.int2ObjectEntrySet()) {
 +            int pIdx = p2tuplesMapEntry.getIntKey();
 +            LSMTreeIndexAccessor lsmAccessor = (LSMTreeIndexAccessor) indexAccessors[pIdx];
 +            IFrameOperationCallback frameOpCallback = frameOpCallbacks[pIdx];
 +            IFrameTupleProcessor processor = processors[pIdx];
-             lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, p2tuplesMapEntry.getValue());
++            lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController,
++                    p2tuplesMapEntry.getValue());
 +        }
  
          writeBuffer.ensureFrameSize(buffer.capacity());
          if (flushedPartialTuples) {
diff --cc asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 8bc2b1ce55,d2dc3cffb5..dd9f4070d2
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@@ -131,20 -110,17 +134,21 @@@ public class LSMPrimaryUpsertOperatorNo
      private final boolean hasMeta;
      private final int filterFieldIndex;
      private final int metaFieldIndex;
 -    protected LockThenSearchOperationCallback searchCallback;
 -    protected IFrameOperationCallback frameOpCallback;
 +    protected final ISearchOperationCallback[] searchCallbacks;
 +    protected final IFrameOperationCallback[] frameOpCallbacks;
 +    protected final ILSMTupleFilterCallback[] tupleFilterCallbacks;
      private final IFrameOperationCallbackFactory frameOpCallbackFactory;
 -    protected AbstractIndexModificationOperationCallback abstractModCallback;
      private final ISearchOperationCallbackFactory searchCallbackFactory;
 -    private final IFrameTupleProcessor processor;
 -    protected LSMTreeIndexAccessor lsmAccessor;
 +    private final IFrameTupleProcessor[] processors;
      private final ITracer tracer;
      private final long traceCategory;
 +    private final ITupleProjector tupleProjector;
-     private long lastRecordInTimeStamp = 0L;
 +    private final Int2ObjectMap<IntSet> partition2TuplesMap = new Int2ObjectOpenHashMap<>();
 +    private final boolean hasSecondaries;
 +    private final ILSMTupleFilterCallbackFactory tupleFilterCallbackFactory;
 +    private final int[] fieldPermutation;
+     private long lastRecordInTimeStamp = 0L;
+     private IBatchController batchController;
  
      public LSMPrimaryUpsertOperatorNodePushable(IHyracksTaskContext ctx, int partition,
              IIndexDataflowHelperFactory indexHelperFactory, int[] fieldPermutation, RecordDescriptor inputRecDesc,
@@@ -376,8 -312,11 +385,9 @@@
                      callback.open();
                  }
              };
 -            frameOpCallback.open();
 -            batchController = TaskUtil.getOrDefault(KEY_BATCH_CONTROLLER, ctx, StandardBatchController.INSTANCE);
 -        } catch (Throwable e) { // NOSONAR: Re-thrown
 -            throw HyracksDataException.create(e);
 +            frameOpCallbacks[i].open();
          }
++        batchController = TaskUtil.getOrDefault(KEY_BATCH_CONTROLLER, ctx, StandardBatchController.INSTANCE);
      }
  
      protected void resetSearchPredicate(int tupleIndex) {
@@@ -417,28 -353,8 +427,29 @@@
      @Override
      public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
          accessor.reset(buffer);
 +        partition2TuplesMap.clear();
          int itemCount = accessor.getTupleCount();
 -        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController);
 +        for (int i = 0; i < itemCount; i++) {
 +            int storagePartition = tuplePartitioner.partition(accessor, i);
 +            if (tupleFilterCallbacks[storagePartitionId2Index.get(storagePartition)].filter(accessor, i)) {
 +                continue;
 +            }
 +            int pIdx = storagePartitionId2Index.get(storagePartition);
 +            IntSet tupleIndexes = partition2TuplesMap.computeIfAbsent(pIdx, k -> new IntOpenHashSet());
 +            tupleIndexes.add(i);
 +        }
 +        // to ensure all partitions will be processed at least once, add partitions with missing tuples
 +        for (int partition : storagePartitionId2Index.values()) {
 +            partition2TuplesMap.computeIfAbsent(partition, k -> new IntOpenHashSet());
 +        }
 +        for (Int2ObjectMap.Entry<IntSet> p2tuplesMapEntry : partition2TuplesMap.int2ObjectEntrySet()) {
 +            int pIdx = p2tuplesMapEntry.getIntKey();
 +            LSMTreeIndexAccessor lsmAccessor = (LSMTreeIndexAccessor) indexAccessors[pIdx];
 +            IFrameOperationCallback frameOpCallback = frameOpCallbacks[pIdx];
 +            IFrameTupleProcessor processor = processors[pIdx];
-             lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, p2tuplesMapEntry.getValue());
++            lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController,
++                    p2tuplesMapEntry.getValue());
 +        }
          if (itemCount > 0) {
              lastRecordInTimeStamp = System.currentTimeMillis();
          }
@@@ -586,42 -495,4 +597,43 @@@
          // No op since nextFrame flushes by default
      }
  
 +    // TODO: Refactor and remove duplicated code
 +    private void commitAtomicUpsert() throws HyracksDataException {
 +        final Map<String, ILSMComponentId> componentIdMap = new HashMap<>();
 +        boolean atomic = false;
 +        for (IIndex index : indexes) {
 +            if (index != null && ((ILSMIndex) index).isAtomic()) {
 +                PrimaryIndexOperationTracker opTracker =
 +                        ((PrimaryIndexOperationTracker) ((ILSMIndex) index).getOperationTracker());
 +                opTracker.finishAllFlush();
 +                for (Map.Entry<String, FlushOperation> entry : opTracker.getLastFlushOperation().entrySet()) {
 +                    componentIdMap.put(entry.getKey(), entry.getValue().getFlushingComponent().getId());
 +                }
 +                atomic = true;
 +            }
 +        }
 +
 +        if (atomic) {
 +            AtomicJobPreparedMessage message = new AtomicJobPreparedMessage(ctx.getJobletContext().getJobId(),
 +                    ctx.getJobletContext().getServiceContext().getNodeId(), componentIdMap);
 +            try {
 +                ((NodeControllerService) ctx.getJobletContext().getServiceContext().getControllerService())
 +                        .sendRealTimeApplicationMessageToCC(ctx.getJobletContext().getJobId().getCcId(),
 +                                JavaSerializationUtils.serialize(message), null);
 +            } catch (Exception e) {
 +                throw new ACIDException(e);
 +            }
 +        }
 +    }
 +
 +    private void abortAtomicUpsert() throws HyracksDataException {
 +        for (IIndex index : indexes) {
 +            if (index != null && ((ILSMIndex) index).isAtomic()) {
 +                PrimaryIndexOperationTracker opTracker =
 +                        ((PrimaryIndexOperationTracker) ((ILSMIndex) index).getOperationTracker());
 +                opTracker.abort();
 +            }
 +        }
 +    }
++
  }
diff --cc hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index 3ed17b138d,abe62c22a1..ad5e91909b
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@@ -288,32 -287,42 +288,68 @@@ public class InvokeUtil 
          }
      }
  
+     @SuppressWarnings({ "squid:S1181", "squid:S1193", "ConstantConditions" }) // catching Throwable, instanceofs
+     public static void tryHyracksWithCleanups(HyracksThrowingAction action, HyracksThrowingAction... cleanups)
+             throws HyracksDataException {
+         Throwable savedT = null;
+         boolean suppressedInterrupted = false;
+         try {
+             action.run();
+         } catch (Throwable t) {
+             savedT = t;
+         } finally {
+             for (HyracksThrowingAction cleanup : cleanups) {
+                 try {
+                     cleanup.run();
+                 } catch (Throwable t) {
+                     if (savedT != null) {
+                         savedT.addSuppressed(t);
+                         suppressedInterrupted = suppressedInterrupted || t instanceof InterruptedException;
+                     } else {
+                         savedT = t;
+                     }
+                 }
+             }
+         }
+         if (savedT == null) {
+             return;
+         }
+         if (suppressedInterrupted) {
+             Thread.currentThread().interrupt();
+         }
+         if (savedT instanceof Error) {
+             throw (Error) savedT;
+         } else {
+             throw HyracksDataException.create(savedT);
+         }
+     }
+ 
 +    public static void tryWithCleanupsUnchecked(Runnable action, Runnable... cleanups) {
 +        Throwable savedT = null;
 +        try {
 +            action.run();
 +        } catch (Throwable t) {
 +            savedT = t;
 +        } finally {
 +            for (Runnable cleanup : cleanups) {
 +                try {
 +                    cleanup.run();
 +                } catch (Throwable t) {
 +                    if (savedT != null) {
 +                        savedT.addSuppressed(t);
 +                    } else {
 +                        savedT = t;
 +                    }
 +                }
 +            }
 +        }
 +        if (savedT instanceof Error) {
 +            throw (Error) savedT;
 +        } else if (savedT != null) {
 +            throw new UncheckedExecutionException(savedT);
 +        }
 +    }
 +
      /**
       * Runs the supplied action, after suspending any pending interruption. An error will be logged if
       * the action is itself interrupted.
diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 214d9dc8dc,68de45ac08..dbf34c9458
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@@ -215,22 -214,21 +215,29 @@@ public interface ILSMHarness 
      /**
       * Perform batch operation on all tuples in the passed frame tuple accessor
       *
 -     * @param ctx             the operation ctx
 -     * @param accessor        the frame tuple accessor
 -     * @param tuple           the mutable tuple used to pass the tuple to the processor
 -     * @param processor       the tuple processor
 -     * @param frameOpCallback the callback at the end of the frame
 +     * @param ctx
 +     *            the operation ctx
 +     * @param accessor
 +     *            the frame tuple accessor
 +     * @param tuple
 +     *            the mutable tuple used to pass the tuple to the processor
 +     * @param processor
 +     *            the tuple processor
 +     * @param frameOpCallback
 +     *            the callback at the end of the frame
 +     * @param tuples
 +     *            the indexes of tuples to process
+      * @param batchController
++     *            the controller of the batch lifecycle
       * @throws HyracksDataException
       */
      void batchOperate(ILSMIndexOperationContext ctx, FrameTupleAccessor accessor, FrameTupleReference tuple,
-             IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, Set<Integer> tuples)
 -            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController)
 -            throws HyracksDataException;
++            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController,
++            Set<Integer> tuples) throws HyracksDataException;
+ 
+     void enter(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException;
+ 
+     void exit(ILSMIndexOperationContext ctx, IFrameOperationCallback callback, boolean success, LSMOperationType op)
              throws HyracksDataException;
  
      /**
diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 2d040204f1,8ad67f7f85..d019a088a1
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@@ -715,13 -724,15 +728,15 @@@ public class LSMHarness implements ILSM
  
      @Override
      public void batchOperate(ILSMIndexOperationContext ctx, FrameTupleAccessor accessor, FrameTupleReference tuple,
-             IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, Set<Integer> tuples)
 -            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController)
--            throws HyracksDataException {
++            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController,
++            Set<Integer> tuples) throws HyracksDataException {
          processor.start();
-         enter(ctx);
+         batchController.batchEnter(ctx, this, frameOpCallback);
+         boolean success = false;
          try {
              try {
 -                processFrame(accessor, tuple, processor);
 +                processFrame(accessor, tuple, processor, tuples);
+                 success = true;
                  frameOpCallback.frameCompleted();
              } catch (Throwable th) {
                  processor.fail(th);
diff --cc hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index fb5984da61,e688727f17..c768768c0a
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@@ -211,8 -211,8 +212,9 @@@ public class LSMTreeIndexAccessor imple
      }
  
      public void batchOperate(FrameTupleAccessor accessor, FrameTupleReference tuple, IFrameTupleProcessor processor,
-             IFrameOperationCallback frameOpCallback, Set<Integer> tuples) throws HyracksDataException {
-         lsmHarness.batchOperate(ctx, accessor, tuple, processor, frameOpCallback, tuples);
 -            IFrameOperationCallback frameOpCallback, IBatchController batchController) throws HyracksDataException {
 -        lsmHarness.batchOperate(ctx, accessor, tuple, processor, frameOpCallback, batchController);
++            IFrameOperationCallback frameOpCallback, IBatchController batchController, Set<Integer> tuples)
++            throws HyracksDataException {
++        lsmHarness.batchOperate(ctx, accessor, tuple, processor, frameOpCallback, batchController, tuples);
      }
  
      @Override


(asterixdb) 13/15: [ASTERIXDB-3503][EXT] Fixing Internal Error issue when Delta table does not exists.

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 5ddcc8f276811f338053249e3ec1b2d4f90ffb4d
Author: ayush.tripathi <ay...@couchbase.com>
AuthorDate: Thu Nov 21 03:20:11 2024 +0530

    [ASTERIXDB-3503][EXT] Fixing Internal Error issue when Delta table does not exists.
    
    - user model changes: yes
    - storage format changes: no
    - interface changes: no
    
    Ext-ref: MB-64314
    
    Change-Id: I8d403c8c0698d9d39dc8988eb8b57588a684dbed
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19098
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Tested-by: Murtadha Hubail <mh...@apache.org>
    Integration-Tests: Murtadha Hubail <mh...@apache.org>
---
 .../deltalake-table-not-exists.00.ddl.sqlpp        | 35 +++++++++++++++++
 .../runtimets/testsuite_external_dataset_s3.xml    |  7 ++++
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  | 44 ++++++++++++++--------
 .../reader/aws/delta/DeltaFileRecordReader.java    | 10 +++--
 .../asterix/external/util/ExternalDataUtils.java   | 30 +++++++++++++++
 .../asterix/external/util/aws/s3/S3Utils.java      |  5 ++-
 6 files changed, 112 insertions(+), 19 deletions(-)

diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-table-not-exists/deltalake-table-not-exists.00.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-table-not-exists/deltalake-table-not-exists.00.ddl.sqlpp
new file mode 100644
index 0000000000..c57de93232
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/deltalake-table-not-exists/deltalake-table-not-exists.00.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+ DROP DATAVERSE test IF EXISTS;
+ CREATE DATAVERSE test;
+
+ USE test;
+
+
+ CREATE TYPE DeltalakeTableType as {
+ };
+
+ CREATE EXTERNAL COLLECTION DeltalakeDataset(DeltalakeTableType) USING %adapter%
+ (
+   %template%,
+   ("container"="playground"),
+   ("definition"="delta-data/s1"),
+   ("table-format" = "delta")
+ );
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index d56c1a46ec..d840527a27 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -339,6 +339,13 @@
         <expected-error>Supported file format for 'delta' tables is 'parquet', but 'avro' was provided.</expected-error>
       </compilation-unit>
     </test-case>
+    <test-case FilePath="external-dataset">
+      <compilation-unit name="common/deltalake-table-not-exists">
+        <placeholder name="adapter" value="S3" />
+        <output-dir compare="Text">none</output-dir>
+        <expected-error>ASX1108: External source error. io.delta.kernel.exceptions.TableNotFoundException: Delta table at path `s3a://playground/delta-data/s1` is not found.</expected-error>
+      </compilation-unit>
+    </test-case>
     <test-case FilePath="external-dataset">
       <compilation-unit name="common/avro/avro-types/avro-map">
         <placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
index 9d93fcea1c..8dc820becb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.external.input.record.reader.aws.delta;
 
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 
 import java.io.IOException;
 import java.io.Serializable;
@@ -33,6 +34,8 @@ import java.util.Set;
 import org.apache.asterix.common.cluster.IClusterStateManager;
 import org.apache.asterix.common.dataflow.ICcApplicationContext;
 import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.exceptions.RuntimeDataException;
 import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
 import org.apache.asterix.external.api.IExternalDataRuntimeContext;
 import org.apache.asterix.external.api.IRecordReader;
@@ -60,6 +63,7 @@ import io.delta.kernel.data.FilteredColumnarBatch;
 import io.delta.kernel.data.Row;
 import io.delta.kernel.defaults.engine.DefaultEngine;
 import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.KernelException;
 import io.delta.kernel.internal.InternalScanFileUtils;
 import io.delta.kernel.types.StructType;
 import io.delta.kernel.utils.CloseableIterator;
@@ -87,20 +91,7 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
             throws AlgebricksException, HyracksDataException {
         this.configuration = configuration;
         Configuration conf = new Configuration();
-        conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
-        conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY, configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
-        if (configuration.get(S3Constants.SESSION_TOKEN_FIELD_NAME) != null) {
-            conf.set(S3Constants.HADOOP_SESSION_TOKEN, configuration.get(S3Constants.SESSION_TOKEN_FIELD_NAME));
-        }
-        conf.set(S3Constants.HADOOP_REGION, configuration.get(S3Constants.REGION_FIELD_NAME));
-        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
-        if (serviceEndpoint != null) {
-            conf.set(S3Constants.HADOOP_SERVICE_END_POINT, serviceEndpoint);
-        }
-        conf.set(ExternalDataConstants.KEY_REQUESTED_FIELDS,
-                configuration.getOrDefault(ExternalDataConstants.KEY_REQUESTED_FIELDS, ""));
-        conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
-                configuration.getOrDefault(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION, ""));
+        configurationBuilder(configuration, conf);
         String tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
                 + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
                 + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
@@ -109,7 +100,13 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
 
         Engine engine = DefaultEngine.create(conf);
         io.delta.kernel.Table table = io.delta.kernel.Table.forPath(engine, tableMetadataPath);
-        Snapshot snapshot = table.getLatestSnapshot(engine);
+        Snapshot snapshot;
+        try {
+            snapshot = table.getLatestSnapshot(engine);
+        } catch (KernelException e) {
+            LOGGER.info("Failed to get latest snapshot for table: {}", tableMetadataPath, e);
+            throw RuntimeDataException.create(ErrorCode.EXTERNAL_SOURCE_ERROR, e, getMessageOrToString(e));
+        }
 
         List<Warning> warnings = new ArrayList<>();
         DeltaConverterContext converterContext = new DeltaConverterContext(configuration, warnings);
@@ -192,6 +189,23 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
     }
 
+    public static void configurationBuilder(Map<String, String> configuration, Configuration conf) {
+        conf.set(S3Constants.HADOOP_ACCESS_KEY_ID, configuration.get(S3Constants.ACCESS_KEY_ID_FIELD_NAME));
+        conf.set(S3Constants.HADOOP_SECRET_ACCESS_KEY, configuration.get(S3Constants.SECRET_ACCESS_KEY_FIELD_NAME));
+        if (configuration.get(S3Constants.SESSION_TOKEN_FIELD_NAME) != null) {
+            conf.set(S3Constants.HADOOP_SESSION_TOKEN, configuration.get(S3Constants.SESSION_TOKEN_FIELD_NAME));
+        }
+        conf.set(S3Constants.HADOOP_REGION, configuration.get(S3Constants.REGION_FIELD_NAME));
+        String serviceEndpoint = configuration.get(SERVICE_END_POINT_FIELD_NAME);
+        if (serviceEndpoint != null) {
+            conf.set(S3Constants.HADOOP_SERVICE_END_POINT, serviceEndpoint);
+        }
+        conf.set(ExternalDataConstants.KEY_REQUESTED_FIELDS,
+                configuration.getOrDefault(ExternalDataConstants.KEY_REQUESTED_FIELDS, ""));
+        conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
+                configuration.getOrDefault(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION, ""));
+    }
+
     @Override
     public IRecordReader<?> createRecordReader(IExternalDataRuntimeContext context) throws HyracksDataException {
         try {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaFileRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaFileRecordReader.java
index 558f8a96fe..a5b21b6cad 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaFileRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/DeltaFileRecordReader.java
@@ -133,9 +133,13 @@ public class DeltaFileRecordReader implements IRecordReader<Row> {
             scanFile = scanFiles.get(fileIndex);
             fileStatus = InternalScanFileUtils.getAddFileStatus(scanFile);
             physicalReadSchema = ScanStateRow.getPhysicalDataReadSchema(engine, scanState);
-            physicalDataIter = engine.getParquetHandler().readParquetFiles(singletonCloseableIterator(fileStatus),
-                    physicalReadSchema, Optional.empty());
-            dataIter = Scan.transformPhysicalData(engine, scanState, scanFile, physicalDataIter);
+            try {
+                physicalDataIter = engine.getParquetHandler().readParquetFiles(singletonCloseableIterator(fileStatus),
+                        physicalReadSchema, Optional.empty());
+                dataIter = Scan.transformPhysicalData(engine, scanState, scanFile, physicalDataIter);
+            } catch (IOException e) {
+                throw HyracksDataException.create(e);
+            }
             return this.hasNext();
         } else {
             return false;
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index b3118702a1..19c197916a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -35,6 +35,7 @@ import static org.apache.asterix.external.util.google.gcs.GCSUtils.validatePrope
 import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.ALL_FIELDS_TYPE;
 import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.EMPTY_TYPE;
 import static org.apache.asterix.runtime.evaluators.functions.StringEvaluatorUtils.RESERVED_REGEX_CHARS;
+import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 import static org.msgpack.core.MessagePack.Code.ARRAY16;
 
 import java.io.ByteArrayOutputStream;
@@ -71,6 +72,7 @@ import org.apache.asterix.external.api.IExternalDataSourceFactory.DataSourceType
 import org.apache.asterix.external.api.IInputStreamFactory;
 import org.apache.asterix.external.api.IRecordReaderFactory;
 import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory.IncludeExcludeMatcher;
+import org.apache.asterix.external.input.record.reader.aws.delta.AwsS3DeltaReaderFactory;
 import org.apache.asterix.external.library.JavaLibrary;
 import org.apache.asterix.external.library.msgpack.MessagePackUtils;
 import org.apache.asterix.external.util.ExternalDataConstants.ParquetOptions;
@@ -109,6 +111,12 @@ import org.apache.iceberg.FileScanTask;
 import org.apache.iceberg.Table;
 import org.apache.iceberg.hadoop.HadoopTables;
 import org.apache.iceberg.io.CloseableIterable;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.exceptions.KernelException;
 
 public class ExternalDataUtils {
 
@@ -117,6 +125,8 @@ public class ExternalDataUtils {
     private static final int DEFAULT_MAX_ARGUMENT_SZ = 1024 * 1024;
     private static final int HEADER_FUDGE = 64;
 
+    private static final Logger LOGGER = LogManager.getLogger();
+
     static {
         valueParserFactoryMap.put(ATypeTag.INTEGER, IntegerParserFactory.INSTANCE);
         valueParserFactoryMap.put(ATypeTag.FLOAT, FloatParserFactory.INSTANCE);
@@ -504,6 +514,26 @@ public class ExternalDataUtils {
         }
     }
 
+    public static void validateDeltaTableExists(Map<String, String> configuration) throws CompilationException {
+        Configuration conf = new Configuration();
+        String tableMetadataPath = null;
+        if (configuration.get(ExternalDataConstants.KEY_EXTERNAL_SOURCE_TYPE)
+                .equals(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3)) {
+            AwsS3DeltaReaderFactory.configurationBuilder(configuration, conf);
+            tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
+                    + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
+                    + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
+        }
+        Engine engine = DefaultEngine.create(conf);
+        io.delta.kernel.Table table = io.delta.kernel.Table.forPath(engine, tableMetadataPath);
+        try {
+            table.getLatestSnapshot(engine);
+        } catch (KernelException e) {
+            LOGGER.info("Failed to get latest snapshot for table: {}", tableMetadataPath, e);
+            throw CompilationException.create(ErrorCode.EXTERNAL_SOURCE_ERROR, e, getMessageOrToString(e));
+        }
+    }
+
     public static void prepareIcebergTableFormat(Map<String, String> configuration, Configuration conf,
             String tableMetadataPath) throws AlgebricksException {
         HadoopTables tables = new HadoopTables(conf);
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 891d7f3bfa..bf0938b961 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -24,6 +24,7 @@ import static org.apache.asterix.common.exceptions.ErrorCode.REQUIRED_PARAM_IF_P
 import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
 import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
 import static org.apache.asterix.external.util.ExternalDataUtils.isDeltaTable;
+import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableExists;
 import static org.apache.asterix.external.util.ExternalDataUtils.validateDeltaTableProperties;
 import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
@@ -281,7 +282,6 @@ public class S3Utils {
         else if (configuration.get(ExternalDataConstants.KEY_FORMAT) == null) {
             throw new CompilationException(ErrorCode.PARAMETERS_REQUIRED, srcLoc, ExternalDataConstants.KEY_FORMAT);
         }
-
         // Both parameters should be passed, or neither should be passed (for anonymous/no auth)
         String accessKeyId = configuration.get(ACCESS_KEY_ID_FIELD_NAME);
         String secretAccessKey = configuration.get(SECRET_ACCESS_KEY_FIELD_NAME);
@@ -346,6 +346,9 @@ public class S3Utils {
         if (!response.sdkHttpResponse().isSuccessful()) {
             throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
         }
+        if (isDeltaTable(configuration)) {
+            validateDeltaTableExists(configuration);
+        }
     }
 
     /**


(asterixdb) 06/15: [NO ISSUE][COMP] Exponential recursion in OperatorManipulationUtil.substituteVarRec method

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 4dde600f6c30259add99cbd47759d08a255aa548
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Tue Nov 19 13:33:06 2024 -0800

    [NO ISSUE][COMP] Exponential recursion in OperatorManipulationUtil.substituteVarRec method
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Ext-ref: MB-64295
    
    Change-Id: Iefe7859bb6b83f59e735d041ef1765e8741d0c5e
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19095
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Peeyush Gupta <pe...@couchbase.com>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
---
 .../rules/PushAggregateIntoNestedSubplanRule.java     |  6 ++++--
 .../optimizer/rules/PushGroupByThroughProduct.java    |  4 +++-
 .../core/algebra/util/OperatorManipulationUtil.java   | 19 +++++++++++++++----
 .../rewriter/rules/AbstractDecorrelationRule.java     |  4 +++-
 .../rules/subplan/IntroduceGroupByForSubplanRule.java |  7 +++++--
 5 files changed, 30 insertions(+), 10 deletions(-)

diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java
index d2ac8e57ec..378c75851d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushAggregateIntoNestedSubplanRule.java
@@ -398,14 +398,16 @@ public class PushAggregateIntoNestedSubplanRule implements IAlgebraicRewriteRule
         if (!pushableNestedSubplan) {
             return false;
         }
-
+        Set<ILogicalOperator> visited = new HashSet<>();
         for (int i = 0; i < nspOp.getNestedPlans().size(); i++) {
             Mutable<ILogicalOperator> nspAggRef = nspOp.getNestedPlans().get(i).getRoots().get(0);
             AggregateOperator nspAgg = (AggregateOperator) nspAggRef.getValue();
             Mutable<ILogicalOperator> nspAggChildRef = nspAgg.getInputs().get(0);
             LogicalVariable listifyVar = findListifiedVariable(nspAgg, varFromNestedAgg);
             if (listifyVar != null) {
-                OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context);
+                OperatorManipulationUtil.substituteVarRec(aggInSubplanOp, unnestVar, listifyVar, true, context,
+                        visited);
+                visited.clear();
                 nspAgg.getVariables().addAll(aggInSubplanOp.getVariables());
                 nspAgg.getExpressions().addAll(aggInSubplanOp.getExpressions());
                 for (LogicalVariable v : aggInSubplanOp.getVariables()) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
index 6d92b515e4..28166a98d5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushGroupByThroughProduct.java
@@ -120,12 +120,14 @@ public class PushGroupByThroughProduct implements IAlgebraicRewriteRule {
         AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) opRefJoin.getValue();
         gby.getDecorList().clear();
         gby.getDecorList().addAll(decorToPush);
+        Set<ILogicalOperator> visited = new HashSet<>();
         for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorNotToPush) {
             LogicalVariable v1 = p.first;
             if (v1 != null) {
                 VariableReferenceExpression varRef = (VariableReferenceExpression) p.second.getValue();
                 LogicalVariable v2 = varRef.getVariableReference();
-                OperatorManipulationUtil.substituteVarRec(join, v2, v1, true, context);
+                OperatorManipulationUtil.substituteVarRec(join, v2, v1, true, context, visited);
+                visited.clear();
             }
         }
         Mutable<ILogicalOperator> branchRef = join.getInputs().get(branch);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
index 2413ed250e..f8fb6c252e 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -181,17 +181,24 @@ public class OperatorManipulationUtil {
     }
 
     public static void substituteVarRec(AbstractLogicalOperator op, LogicalVariable v1, LogicalVariable v2,
-            boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
+            boolean goThroughNts, ITypingContext ctx, Set<ILogicalOperator> visited) throws AlgebricksException {
         VariableUtilities.substituteVariables(op, v1, v2, goThroughNts, ctx);
         for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
-            substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx);
+            if (visited.contains(opRef2.getValue())) {
+                continue;
+            }
+            visited.add(opRef2.getValue());
+            substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx, visited);
         }
         if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE && goThroughNts) {
             NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
             if (nts.getDataSourceReference() != null) {
                 AbstractLogicalOperator op2 =
                         (AbstractLogicalOperator) nts.getDataSourceReference().getValue().getInputs().get(0).getValue();
-                substituteVarRec(op2, v1, v2, goThroughNts, ctx);
+                if (!visited.contains(op2)) {
+                    visited.add(op2);
+                    substituteVarRec(op2, v1, v2, goThroughNts, ctx, visited);
+                }
             }
         }
         if (op.hasNestedPlans()) {
@@ -199,7 +206,11 @@ public class OperatorManipulationUtil {
             for (ILogicalPlan p : aonp.getNestedPlans()) {
                 for (Mutable<ILogicalOperator> ref : p.getRoots()) {
                     AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getValue();
-                    substituteVarRec(aop, v1, v2, goThroughNts, ctx);
+                    if (visited.contains(aop)) {
+                        continue;
+                    }
+                    visited.add(aop);
+                    substituteVarRec(aop, v1, v2, goThroughNts, ctx, visited);
                 }
             }
         }
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
index 55c8400e3d..dbdf6c48a3 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
@@ -92,6 +92,7 @@ public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule
     protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g,
             List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
         SourceLocation sourceLoc = g.getSourceLocation();
+        Set<ILogicalOperator> visited = new HashSet<>();
         for (LogicalVariable ov : vars) {
             LogicalVariable newVar = context.newVar();
             VariableReferenceExpression varExpr = new VariableReferenceExpression(newVar);
@@ -101,7 +102,8 @@ public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule
             for (ILogicalPlan p : g.getNestedPlans()) {
                 for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
-                            context);
+                            context, visited);
+                    visited.clear();
                 }
             }
         }
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/IntroduceGroupByForSubplanRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/IntroduceGroupByForSubplanRule.java
index 391d03a8d8..36cad77208 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/IntroduceGroupByForSubplanRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/subplan/IntroduceGroupByForSubplanRule.java
@@ -331,6 +331,7 @@ public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
             List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
         SourceLocation sourceLoc = g.getSourceLocation();
         Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
+        Set<ILogicalOperator> visited = new HashSet<>();
         for (LogicalVariable ov : vars) {
             LogicalVariable newVar = context.newVar();
             ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
@@ -340,11 +341,13 @@ public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
             for (ILogicalPlan p : g.getNestedPlans()) {
                 for (Mutable<ILogicalOperator> r : p.getRoots()) {
                     OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
-                            context);
+                            context, visited);
+                    visited.clear();
                 }
             }
             AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
-            OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
+            OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context, visited);
+            visited.clear();
             m.put(ov, newVar);
         }
         return m;


(asterixdb) 12/15: Merge branch 'gerrit/trinity' into 'gerrit/goldfish'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit bdaf4e97e4373b4c5afbb7ffb6f0aeb1f24cd9fa
Merge: e9772ce511 52b8df6313
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Sat Nov 23 09:57:47 2024 -0500

    Merge branch 'gerrit/trinity' into 'gerrit/goldfish'
    
    Ext-ref: MB-64326,MB-64295
    Change-Id: I3a68fd33690f54d392782658e15eff8ba804d964

 .../rules/PushAggregateIntoNestedSubplanRule.java     |  6 ++++--
 .../optimizer/rules/PushGroupByThroughProduct.java    |  4 +++-
 .../core/algebra/util/OperatorManipulationUtil.java   | 19 +++++++++++++++----
 .../rewriter/rules/AbstractDecorrelationRule.java     |  4 +++-
 .../rules/subplan/IntroduceGroupByForSubplanRule.java |  7 +++++--
 5 files changed, 30 insertions(+), 10 deletions(-)


(asterixdb) 11/15: [NO ISSUE]: Support Reading credentials from String

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit e9772ce511d043d8490d83a65f1210694dbc0eb3
Author: Savyasach Reddy <es...@gmail.com>
AuthorDate: Mon Nov 18 13:52:58 2024 +0530

    [NO ISSUE]: Support Reading credentials from String
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    details:
    - Read JSON credentials from a string instead of a file to read
      Parquet data from Google Cloud Storage
    
    Ext-ref: MB-64268
    Change-Id: Iac78b7df9688e134f2fc7e2eb737b909a71077e0
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19084
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Tested-by: Murtadha Hubail <mh...@apache.org>
    Integration-Tests: Murtadha Hubail <mh...@apache.org>
---
 .../external/util/google/gcs/GCSConstants.java     | 12 ++++++++
 .../asterix/external/util/google/gcs/GCSUtils.java | 35 ++++++++++++++++------
 2 files changed, 38 insertions(+), 9 deletions(-)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
index f2dbde7f80..2818043179 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSConstants.java
@@ -44,4 +44,16 @@ public class GCSConstants {
     public static final String HADOOP_ENDPOINT = "fs.gs.storage.root.url";
     public static final String HADOOP_MAX_REQUESTS_PER_BATCH = "fs.gs.max.requests.per.batch";
     public static final String HADOOP_BATCH_THREADS = "fs.gs.batch.threads";
+
+    public static class JSON_CREDENTIALS_FIELDS {
+        public static final String PRIVATE_KEY_ID = "private_key_id";
+        public static final String PRIVATE_KEY = "private_key";
+        public static final String CLIENT_EMAIL = "client_email";
+    }
+
+    public static class HADOOP_AUTH_SERVICE_ACCOUNT_JSON_FIELDS {
+        public static final String PRIVATE_KEY_ID = "fs.gs.auth.service.account.private.key.id";
+        public static final String PRIVATE_KEY = "fs.gs.auth.service.account.private.key";
+        public static final String CLIENT_EMAIL = "fs.gs.auth.service.account.email";
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
index 5274c44a38..74a664da35 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/google/gcs/GCSUtils.java
@@ -25,8 +25,6 @@ import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
 import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
 import static org.apache.asterix.external.util.google.gcs.GCSConstants.APPLICATION_DEFAULT_CREDENTIALS_FIELD_NAME;
 import static org.apache.asterix.external.util.google.gcs.GCSConstants.ENDPOINT_FIELD_NAME;
-import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE;
-import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH;
 import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_TYPE;
 import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_AUTH_UNAUTHENTICATED;
 import static org.apache.asterix.external.util.google.gcs.GCSConstants.HADOOP_ENDPOINT;
@@ -37,7 +35,6 @@ import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -59,6 +56,10 @@ import org.apache.hyracks.api.exceptions.IWarningCollector;
 import org.apache.hyracks.api.exceptions.SourceLocation;
 import org.apache.hyracks.api.exceptions.Warning;
 
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.json.JsonReadFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.api.gax.paging.Page;
 import com.google.auth.oauth2.GoogleCredentials;
 import com.google.cloud.BaseServiceException;
@@ -73,6 +74,11 @@ public class GCSUtils {
 
     }
 
+    private static final ObjectMapper JSON_CREDENTIALS_OBJECT_MAPPER = new ObjectMapper();
+    static {
+        JSON_CREDENTIALS_OBJECT_MAPPER.configure(JsonReadFeature.ALLOW_UNESCAPED_CONTROL_CHARS.mappedFeature(), true);
+    }
+
     /**
      * Builds the client using the provided configuration
      *
@@ -218,7 +224,8 @@ public class GCSUtils {
      * @param configuration      properties
      * @param numberOfPartitions number of partitions in the cluster
      */
-    public static void configureHdfsJobConf(JobConf conf, Map<String, String> configuration, int numberOfPartitions) {
+    public static void configureHdfsJobConf(JobConf conf, Map<String, String> configuration, int numberOfPartitions)
+            throws AlgebricksException {
         String jsonCredentials = configuration.get(JSON_CREDENTIALS_FIELD_NAME);
         String endpoint = configuration.get(ENDPOINT_FIELD_NAME);
 
@@ -234,15 +241,25 @@ public class GCSUtils {
         //        conf.set(GCSConstants.HADOOP_BATCH_THREADS, String.valueOf(numberOfPartitions));
 
         // authentication method
-        // TODO(htowaileb): find a way to pass the content instead of the path to keyfile, this line is temporary
-        Path credentials = Path.of("credentials.json");
         if (jsonCredentials == null) {
             // anonymous access
             conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_UNAUTHENTICATED);
         } else {
-            // TODO(htowaileb) need to pass the file content
-            conf.set(HADOOP_AUTH_TYPE, HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE);
-            conf.set(HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH, credentials.toAbsolutePath().toString());
+            try {
+                JsonNode jsonCreds = JSON_CREDENTIALS_OBJECT_MAPPER.readTree(jsonCredentials);
+                // Setting these values instead of HADOOP_AUTH_SERVICE_ACCOUNT_JSON_KEY_FILE_PATH is supported
+                // in com.google.cloud.bigdataoss:util-hadoop only up to version hadoop3-2.2.x and is removed in
+                // version 3.x.y, which also removed support for hadoop-2
+                conf.set(GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_FIELDS.PRIVATE_KEY_ID,
+                        jsonCreds.get(GCSConstants.JSON_CREDENTIALS_FIELDS.PRIVATE_KEY_ID).asText());
+                conf.set(GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_FIELDS.PRIVATE_KEY,
+                        jsonCreds.get(GCSConstants.JSON_CREDENTIALS_FIELDS.PRIVATE_KEY).asText());
+                conf.set(GCSConstants.HADOOP_AUTH_SERVICE_ACCOUNT_JSON_FIELDS.CLIENT_EMAIL,
+                        jsonCreds.get(GCSConstants.JSON_CREDENTIALS_FIELDS.CLIENT_EMAIL).asText());
+            } catch (JsonProcessingException e) {
+                throw CompilationException.create(EXTERNAL_SOURCE_ERROR, "Unable to parse Json Credentials",
+                        getMessageOrToString(e));
+            }
         }
 
         // set endpoint if provided, default is https://storage.googleapis.com/


(asterixdb) 10/15: Merge branch 'gerrit/neo' into 'gerrit/trinity'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 52b8df6313f62da85fe55351b1e300861469d1d5
Merge: 33db60a5ae 4dde600f6c
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Thu Nov 21 15:24:47 2024 -0500

    Merge branch 'gerrit/neo' into 'gerrit/trinity'
    
    Ext-ref: MB-63896
    Change-Id: I4352f491f0dec61b34ad1e559bf2a8d1de80f264

 .../rules/PushAggregateIntoNestedSubplanRule.java     |  6 ++++--
 .../optimizer/rules/PushGroupByThroughProduct.java    |  4 +++-
 .../core/algebra/util/OperatorManipulationUtil.java   | 19 +++++++++++++++----
 .../rewriter/rules/AbstractDecorrelationRule.java     |  4 +++-
 .../rules/subplan/IntroduceGroupByForSubplanRule.java |  7 +++++--
 5 files changed, 30 insertions(+), 10 deletions(-)


(asterixdb) 01/15: [NO ISSUE][HYR][STO,MISC] LSM enhancements, misc utilities

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit f11758e0d86d07e687f21eb432e1ea2d0bb08c3e
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Fri Nov 15 06:26:05 2024 -0500

    [NO ISSUE][HYR][STO,MISC] LSM enhancements, misc utilities
    
    - provide ability to intercept entering / exiting components on
      batch LSM operations
    - += HyracksThrowingAction, InvokeUtil.tryHyracksWithCleanups
    
    Ext-ref: MB-64229
    Change-Id: I03f573d44b170f0c4d889920a3991592fb2890e1
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19072
    Reviewed-by: Michael Blow <mb...@apache.org>
    Reviewed-by: Ali Alsuliman <al...@gmail.com>
    Tested-by: Michael Blow <mb...@apache.org>
---
 .../LSMPrimaryInsertOperatorNodePushable.java      | 10 +++++-
 .../LSMPrimaryUpsertOperatorNodePushable.java      |  8 ++++-
 .../runtime/operators/StandardBatchController.java | 42 ++++++++++++++++++++++
 .../apache/hyracks/api/comm/IFrameAppender.java    |  2 +-
 .../hyracks/api/util/HyracksThrowingAction.java    | 26 ++++++++++++++
 .../org/apache/hyracks/api/util/InvokeUtil.java    | 36 +++++++++++++++++++
 .../common/io/MessagingFrameTupleAppender.java     |  9 -----
 .../hyracks/dataflow/common/utils/TaskUtil.java    | 14 ++++++++
 .../am/lsm/common/api/IBatchController.java        | 29 +++++++++++++++
 .../storage/am/lsm/common/api/ILSMHarness.java     | 34 +++++++-----------
 .../storage/am/lsm/common/impls/LSMHarness.java    | 18 ++++++----
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |  5 +--
 12 files changed, 190 insertions(+), 43 deletions(-)

diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
index eadb614799..5c879944fd 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryInsertOperatorNodePushable.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.runtime.operators;
 
+import static org.apache.hyracks.storage.am.lsm.common.api.IBatchController.KEY_BATCH_CONTROLLER;
+
 import java.nio.ByteBuffer;
 
 import org.apache.asterix.common.api.INcApplicationContext;
@@ -33,6 +35,7 @@ import org.apache.hyracks.api.exceptions.ErrorCode;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.SourceLocation;
 import org.apache.hyracks.api.util.CleanupUtils;
+import org.apache.hyracks.api.util.HyracksThrowingAction;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
 import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAppender;
 import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
@@ -50,6 +53,7 @@ import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory
 import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
 import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor;
 import org.apache.hyracks.storage.am.lsm.common.dataflow.LSMIndexInsertUpdateDeleteOperatorNodePushable;
@@ -77,6 +81,7 @@ public class LSMPrimaryInsertOperatorNodePushable extends LSMIndexInsertUpdateDe
     private boolean flushedPartialTuples;
     private int currentTupleIdx;
     private int lastFlushedTupleIdx;
+    private IBatchController batchController;
 
     private final PermutingFrameTupleReference keyTuple;
 
@@ -116,6 +121,8 @@ public class LSMPrimaryInsertOperatorNodePushable extends LSMIndexInsertUpdateDe
 
     protected IFrameTupleProcessor createTupleProcessor(SourceLocation sourceLoc) {
         return new IFrameTupleProcessor() {
+            private HyracksThrowingAction exitAction;
+
             @Override
             public void process(ITupleReference tuple, int index) throws HyracksDataException {
                 if (index < currentTupleIdx) {
@@ -219,6 +226,7 @@ public class LSMPrimaryInsertOperatorNodePushable extends LSMIndexInsertUpdateDe
                     (INcApplicationContext) ctx.getJobletContext().getServiceContext().getApplicationContext();
             LSMIndexUtil.checkAndSetFirstLSN((AbstractLSMIndex) index,
                     appCtx.getTransactionSubsystem().getLogManager());
+            batchController = TaskUtil.getOrDefault(KEY_BATCH_CONTROLLER, ctx, StandardBatchController.INSTANCE);
         } catch (Throwable e) { // NOSONAR: Re-thrown
             throw HyracksDataException.create(e);
         }
@@ -227,7 +235,7 @@ public class LSMPrimaryInsertOperatorNodePushable extends LSMIndexInsertUpdateDe
     @Override
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
         accessor.reset(buffer);
-        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback);
+        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController);
 
         writeBuffer.ensureFrameSize(buffer.capacity());
         if (flushedPartialTuples) {
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 3b8ee68370..643c6abef1 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -18,6 +18,8 @@
  */
 package org.apache.asterix.runtime.operators;
 
+import static org.apache.hyracks.storage.am.lsm.common.api.IBatchController.KEY_BATCH_CONTROLLER;
+
 import java.io.DataOutput;
 import java.io.IOException;
 import java.nio.ByteBuffer;
@@ -62,6 +64,7 @@ import org.apache.hyracks.storage.am.common.api.ITreeIndex;
 import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
 import org.apache.hyracks.storage.am.common.impls.IndexAccessParameters;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallbackFactory;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor;
@@ -117,6 +120,7 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe
     private final ITracer tracer;
     private final long traceCategory;
     private long lastRecordInTimeStamp = 0L;
+    private IBatchController batchController;
 
     public LSMPrimaryUpsertOperatorNodePushable(IHyracksTaskContext ctx, int partition,
             IIndexDataflowHelperFactory indexHelperFactory, int[] fieldPermutation, RecordDescriptor inputRecDesc,
@@ -304,6 +308,7 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe
                 }
             };
             frameOpCallback.open();
+            batchController = TaskUtil.getOrDefault(KEY_BATCH_CONTROLLER, ctx, StandardBatchController.INSTANCE);
         } catch (Throwable e) { // NOSONAR: Re-thrown
             throw HyracksDataException.create(e);
         }
@@ -344,7 +349,7 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe
     public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
         accessor.reset(buffer);
         int itemCount = accessor.getTupleCount();
-        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback);
+        lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback, batchController);
         if (itemCount > 0) {
             lastRecordInTimeStamp = System.currentTimeMillis();
         }
@@ -484,4 +489,5 @@ public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDe
     public void flush() throws HyracksDataException {
         // No op since nextFrame flushes by default
     }
+
 }
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
new file mode 100644
index 0000000000..f9f758b64f
--- /dev/null
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/StandardBatchController.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.runtime.operators;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMHarness;
+import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
+import org.apache.hyracks.storage.am.lsm.common.api.LSMOperationType;
+
+class StandardBatchController implements IBatchController {
+    static final IBatchController INSTANCE = new StandardBatchController();
+
+    private StandardBatchController() {
+    }
+
+    @Override
+    public void batchEnter(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException {
+        lsmHarness.enter(ctx, LSMOperationType.MODIFICATION);
+    }
+
+    @Override
+    public void batchExit(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException {
+        lsmHarness.exit(ctx, LSMOperationType.MODIFICATION);
+    }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/IFrameAppender.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/IFrameAppender.java
index 0fc24c7ea2..98215bf738 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/IFrameAppender.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/comm/IFrameAppender.java
@@ -63,7 +63,7 @@ public interface IFrameAppender {
      * @param writer the FrameWriter to write to and flush
      * @throws HyracksDataException
      */
-    public default void flush(IFrameWriter writer) throws HyracksDataException {
+    default void flush(IFrameWriter writer) throws HyracksDataException {
         write(writer, true);
         writer.flush();
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingAction.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingAction.java
new file mode 100644
index 0000000000..7e3d5997b0
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/HyracksThrowingAction.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.api.util;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+@FunctionalInterface
+public interface HyracksThrowingAction {
+    void run() throws HyracksDataException; // NOSONAR
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
index f4b6e20858..d331ab2135 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/InvokeUtil.java
@@ -285,6 +285,42 @@ public class InvokeUtil {
         }
     }
 
+    @SuppressWarnings({ "squid:S1181", "squid:S1193", "ConstantConditions" }) // catching Throwable, instanceofs
+    public static void tryHyracksWithCleanups(HyracksThrowingAction action, HyracksThrowingAction... cleanups)
+            throws HyracksDataException {
+        Throwable savedT = null;
+        boolean suppressedInterrupted = false;
+        try {
+            action.run();
+        } catch (Throwable t) {
+            savedT = t;
+        } finally {
+            for (HyracksThrowingAction cleanup : cleanups) {
+                try {
+                    cleanup.run();
+                } catch (Throwable t) {
+                    if (savedT != null) {
+                        savedT.addSuppressed(t);
+                        suppressedInterrupted = suppressedInterrupted || t instanceof InterruptedException;
+                    } else {
+                        savedT = t;
+                    }
+                }
+            }
+        }
+        if (savedT == null) {
+            return;
+        }
+        if (suppressedInterrupted) {
+            Thread.currentThread().interrupt();
+        }
+        if (savedT instanceof Error) {
+            throw (Error) savedT;
+        } else {
+            throw HyracksDataException.create(savedT);
+        }
+    }
+
     /**
      * Runs the supplied action, after suspending any pending interruption. An error will be logged if
      * the action is itself interrupted.
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/MessagingFrameTupleAppender.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/MessagingFrameTupleAppender.java
index c27a7e68dd..6c073f3bce 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/MessagingFrameTupleAppender.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/io/MessagingFrameTupleAppender.java
@@ -172,13 +172,4 @@ public class MessagingFrameTupleAppender extends FrameTupleAppender {
         ++tupleCount;
         IntSerDeUtils.putInt(getBuffer().array(), FrameHelper.getTupleCountOffset(frame.getFrameSize()), tupleCount);
     }
-
-    /*
-     * Always write and then flush to send out the message if exists
-     */
-    @Override
-    public void flush(IFrameWriter writer) throws HyracksDataException {
-        write(writer, true);
-        writer.flush();
-    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/utils/TaskUtil.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/utils/TaskUtil.java
index 75c95b0053..b77883d254 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/utils/TaskUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/utils/TaskUtil.java
@@ -74,4 +74,18 @@ public class TaskUtil {
         Map<String, Object> sharedMap = TaskUtil.getSharedMap(ctx, false);
         return sharedMap == null ? null : (T) sharedMap.get(key);
     }
+
+    /**
+     * get a <T> object from the shared map of the task, or returns the default value
+     *
+     * @param key
+     * @param ctx
+     * @param defaultValue
+     * @return the value associated with the key casted as T
+     */
+    @SuppressWarnings("unchecked")
+    public static <T> T getOrDefault(String key, IHyracksTaskContext ctx, T defaultValue) {
+        Map<String, T> sharedMap = (Map<String, T>) TaskUtil.getSharedMap(ctx, false);
+        return sharedMap == null ? defaultValue : sharedMap.getOrDefault(key, defaultValue);
+    }
 }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java
new file mode 100644
index 0000000000..879a8d223b
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/IBatchController.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.storage.am.lsm.common.api;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface IBatchController {
+    String KEY_BATCH_CONTROLLER = "BATCH_CONTROLLER";
+
+    void batchEnter(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException;
+
+    void batchExit(ILSMHarness lsmHarness, ILSMIndexOperationContext ctx) throws HyracksDataException;
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
index 9e8c568bc6..721d809c64 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/api/ILSMHarness.java
@@ -39,7 +39,6 @@ public interface ILSMHarness {
      * @param tuple
      *            the operation tuple
      * @throws HyracksDataException
-     * @throws IndexException
      */
     void forceModify(ILSMIndexOperationContext ctx, ITupleReference tuple) throws HyracksDataException;
 
@@ -54,7 +53,6 @@ public interface ILSMHarness {
      *            the operation tuple
      * @return
      * @throws HyracksDataException
-     * @throws IndexException
      */
     boolean modify(ILSMIndexOperationContext ctx, boolean tryOperation, ITupleReference tuple)
             throws HyracksDataException;
@@ -69,7 +67,6 @@ public interface ILSMHarness {
      * @param pred
      *            the search predicate
      * @throws HyracksDataException
-     * @throws IndexException
      */
     void search(ILSMIndexOperationContext ctx, IIndexCursor cursor, ISearchPredicate pred) throws HyracksDataException;
 
@@ -104,9 +101,7 @@ public interface ILSMHarness {
      * Schedule a merge
      *
      * @param ctx
-     * @param callback
      * @throws HyracksDataException
-     * @throws IndexException
      */
     ILSMIOOperation scheduleMerge(ILSMIndexOperationContext ctx) throws HyracksDataException;
 
@@ -114,9 +109,7 @@ public interface ILSMHarness {
      * Schedule full merge
      *
      * @param ctx
-     * @param callback
      * @throws HyracksDataException
-     * @throws IndexException
      */
     ILSMIOOperation scheduleFullMerge(ILSMIndexOperationContext ctx) throws HyracksDataException;
 
@@ -125,7 +118,6 @@ public interface ILSMHarness {
      *
      * @param operation
      * @throws HyracksDataException
-     * @throws IndexException
      */
     void merge(ILSMIOOperation operation) throws HyracksDataException;
 
@@ -133,7 +125,6 @@ public interface ILSMHarness {
      * Schedule a flush
      *
      * @param ctx
-     * @param callback
      * @throws HyracksDataException
      */
     ILSMIOOperation scheduleFlush(ILSMIndexOperationContext ctx) throws HyracksDataException;
@@ -143,7 +134,6 @@ public interface ILSMHarness {
      *
      * @param operation
      * @throws HyracksDataException
-     * @throws IndexException
      */
     void flush(ILSMIOOperation operation) throws HyracksDataException;
 
@@ -153,7 +143,6 @@ public interface ILSMHarness {
      * @param ioOperation
      *            the io operation that added the new component
      * @throws HyracksDataException
-     * @throws IndexException
      */
     void addBulkLoadedComponent(ILSMIOOperation ioOperation) throws HyracksDataException;
 
@@ -225,20 +214,21 @@ public interface ILSMHarness {
     /**
      * Perform batch operation on all tuples in the passed frame tuple accessor
      *
-     * @param ctx
-     *            the operation ctx
-     * @param accessor
-     *            the frame tuple accessor
-     * @param tuple
-     *            the mutable tuple used to pass the tuple to the processor
-     * @param processor
-     *            the tuple processor
-     * @param frameOpCallback
-     *            the callback at the end of the frame
+     * @param ctx             the operation ctx
+     * @param accessor        the frame tuple accessor
+     * @param tuple           the mutable tuple used to pass the tuple to the processor
+     * @param processor       the tuple processor
+     * @param frameOpCallback the callback at the end of the frame
+     * @param batchController
      * @throws HyracksDataException
      */
     void batchOperate(ILSMIndexOperationContext ctx, FrameTupleAccessor accessor, FrameTupleReference tuple,
-            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback) throws HyracksDataException;
+            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController)
+            throws HyracksDataException;
+
+    void enter(ILSMIndexOperationContext ctx, LSMOperationType opType) throws HyracksDataException;
+
+    void exit(ILSMIndexOperationContext ctx, LSMOperationType op) throws HyracksDataException;
 
     /**
      * Rollback components that match the passed predicate
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
index 950a8e5b23..6e206869b1 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java
@@ -33,6 +33,7 @@ import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 import org.apache.hyracks.storage.am.common.impls.NoOpIndexAccessParameters;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -688,15 +689,17 @@ public class LSMHarness implements ILSMHarness {
         lsmIndex.updateFilter(ctx, tuple);
     }
 
-    private void enter(ILSMIndexOperationContext ctx) throws HyracksDataException {
+    @Override
+    public void enter(ILSMIndexOperationContext ctx, LSMOperationType op) throws HyracksDataException {
         if (!lsmIndex.isMemoryComponentsAllocated()) {
             lsmIndex.allocateMemoryComponents();
         }
-        getAndEnterComponents(ctx, LSMOperationType.MODIFICATION, false);
+        getAndEnterComponents(ctx, op, false);
     }
 
-    private void exit(ILSMIndexOperationContext ctx) throws HyracksDataException {
-        getAndExitComponentsAndComplete(ctx, LSMOperationType.MODIFICATION);
+    @Override
+    public void exit(ILSMIndexOperationContext ctx, LSMOperationType op) throws HyracksDataException {
+        getAndExitComponentsAndComplete(ctx, op);
     }
 
     private void getAndExitComponentsAndComplete(ILSMIndexOperationContext ctx, LSMOperationType op)
@@ -711,9 +714,10 @@ public class LSMHarness implements ILSMHarness {
 
     @Override
     public void batchOperate(ILSMIndexOperationContext ctx, FrameTupleAccessor accessor, FrameTupleReference tuple,
-            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback) throws HyracksDataException {
+            IFrameTupleProcessor processor, IFrameOperationCallback frameOpCallback, IBatchController batchController)
+            throws HyracksDataException {
         processor.start();
-        enter(ctx);
+        batchController.batchEnter(this, ctx);
         try {
             try {
                 processFrame(accessor, tuple, processor);
@@ -728,7 +732,7 @@ public class LSMHarness implements ILSMHarness {
             LOGGER.warn("Failed to process frame", e);
             throw e;
         } finally {
-            exit(ctx);
+            batchController.batchExit(this, ctx);
             ctx.logPerformanceCounters(accessor.getTupleCount());
         }
     }
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
index 8412b8c648..e688727f17 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMTreeIndexAccessor.java
@@ -28,6 +28,7 @@ import org.apache.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
 import org.apache.hyracks.dataflow.common.data.accessors.FrameTupleReference;
 import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
 import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
+import org.apache.hyracks.storage.am.lsm.common.api.IBatchController;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameOperationCallback;
 import org.apache.hyracks.storage.am.lsm.common.api.IFrameTupleProcessor;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent;
@@ -210,8 +211,8 @@ public class LSMTreeIndexAccessor implements ILSMIndexAccessor {
     }
 
     public void batchOperate(FrameTupleAccessor accessor, FrameTupleReference tuple, IFrameTupleProcessor processor,
-            IFrameOperationCallback frameOpCallback) throws HyracksDataException {
-        lsmHarness.batchOperate(ctx, accessor, tuple, processor, frameOpCallback);
+            IFrameOperationCallback frameOpCallback, IBatchController batchController) throws HyracksDataException {
+        lsmHarness.batchOperate(ctx, accessor, tuple, processor, frameOpCallback, batchController);
     }
 
     @Override


(asterixdb) 05/15: Merge branch 'gerrit/neo' into 'gerrit/trinity'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 33db60a5ae619c0f2812d269ba0a62c2eb162e26
Merge: e5236fd0d9 096385e346
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Wed Nov 20 15:28:28 2024 -0500

    Merge branch 'gerrit/neo' into 'gerrit/trinity'
    
    Ext-ref: MB-64229
    Change-Id: Ic3fd8f7abd64252ad2473b049a0d222747a43857

 .../NoOpFrameOperationCallbackFactory.java         |  5 +++
 .../LSMPrimaryInsertOperatorNodePushable.java      | 10 ++++-
 .../LSMPrimaryUpsertOperatorNodePushable.java      | 13 ++++++-
 .../runtime/operators/StandardBatchController.java | 45 ++++++++++++++++++++++
 .../apache/hyracks/api/comm/IFrameAppender.java    |  2 +-
 .../hyracks/api/util/HyracksThrowingAction.java}   | 30 ++-------------
 .../org/apache/hyracks/api/util/InvokeUtil.java    | 36 +++++++++++++++++
 .../common/io/MessagingFrameTupleAppender.java     |  9 -----
 .../hyracks/dataflow/common/utils/TaskUtil.java    | 14 +++++++
 ...perationCallback.java => IBatchController.java} | 29 +++-----------
 .../am/lsm/common/api/IFrameOperationCallback.java | 14 ++++++-
 .../storage/am/lsm/common/api/ILSMHarness.java     | 35 +++++++----------
 .../storage/am/lsm/common/impls/LSMHarness.java    | 30 +++++++++++----
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |  5 ++-
 14 files changed, 183 insertions(+), 94 deletions(-)



(asterixdb) 03/15: [ASTERIXDB-3503][EXT] Improve logic of distributing files to partitions

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 49929014b3c8e69d37d0eb21e7b9bbb459663de2
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Tue Nov 19 09:54:06 2024 -0800

    [ASTERIXDB-3503][EXT] Improve logic of distributing files to partitions
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    With this patch the files to scan are distributed among different
    partitions such that the total size of files to read is distributed
    fairly among different partitions.
    
    Ext-ref: MB-63840
    Change-Id: Id2c56b707f6fd5f4cb40f4576dc12ceb2e61a193
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19093
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
---
 .../aws/delta/AsterixDeltaRuntimeException.java    | 34 ++++++++++
 .../aws/delta/AsterixTypeToDeltaTypeVisitor.java   |  2 +-
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  | 77 ++++++++++++++++------
 .../asterix/external/util/ExternalDataUtils.java   |  2 +-
 4 files changed, 94 insertions(+), 21 deletions(-)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixDeltaRuntimeException.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixDeltaRuntimeException.java
new file mode 100644
index 0000000000..9dd2000925
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixDeltaRuntimeException.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.aws.delta;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class AsterixDeltaRuntimeException extends RuntimeException {
+    private static final long serialVersionUID = 1L;
+    private final HyracksDataException hyracksDataException;
+
+    public AsterixDeltaRuntimeException(HyracksDataException e) {
+        this.hyracksDataException = e;
+    }
+
+    public HyracksDataException getHyracksDataException() {
+        return hyracksDataException;
+    }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
index 2c86132490..3ca704ff40 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
@@ -124,7 +124,7 @@ public class AsterixTypeToDeltaTypeVisitor implements IATypeVisitor<DataType, Da
         try {
             actualType = DeltaDataParser.getTypeTag(type, false, context);
         } catch (HyracksDataException e) {
-            throw new RuntimeException(e);
+            throw new AsterixDeltaRuntimeException(e);
         }
         ATypeTag expectedType = node.getTypeTag();
 
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
index 5ce2c78152..9d93fcea1c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -21,11 +21,13 @@ package org.apache.asterix.external.input.record.reader.aws.delta;
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
 
 import java.io.IOException;
+import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.PriorityQueue;
 import java.util.Set;
 
 import org.apache.asterix.common.cluster.IClusterStateManager;
@@ -58,8 +60,10 @@ import io.delta.kernel.data.FilteredColumnarBatch;
 import io.delta.kernel.data.Row;
 import io.delta.kernel.defaults.engine.DefaultEngine;
 import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
 import io.delta.kernel.types.StructType;
 import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
 
 public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
 
@@ -68,10 +72,9 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
             Collections.singletonList(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
     private static final Logger LOGGER = LogManager.getLogger();
     private transient AlgebricksAbsolutePartitionConstraint locationConstraints;
-    private Map<Integer, List<String>> schedule;
     private String scanState;
     private Map<String, String> configuration;
-    private List<String> scanFiles;
+    protected final List<PartitionWorkLoadBasedOnSize> partitionWorkLoadsBasedOnSize = new ArrayList<>();
 
     @Override
     public AlgebricksAbsolutePartitionConstraint getPartitionConstraint() {
@@ -120,28 +123,30 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
             requiredSchema = visitor.clipType(expectedType, fileSchema, functionCallInformationMap);
         } catch (IOException e) {
             throw new RuntimeException(e);
+        } catch (AsterixDeltaRuntimeException e) {
+            throw e.getHyracksDataException();
         }
         Scan scan = snapshot.getScanBuilder(engine).withReadSchema(engine, requiredSchema).build();
         scanState = RowSerDe.serializeRowToJson(scan.getScanState(engine));
         CloseableIterator<FilteredColumnarBatch> iter = scan.getScanFiles(engine);
 
-        scanFiles = new ArrayList<>();
+        List<Row> scanFiles = new ArrayList<>();
         while (iter.hasNext()) {
             FilteredColumnarBatch batch = iter.next();
             CloseableIterator<Row> rowIter = batch.getRows();
             while (rowIter.hasNext()) {
                 Row row = rowIter.next();
-                scanFiles.add(RowSerDe.serializeRowToJson(row));
+                scanFiles.add(row);
             }
         }
-        locationConstraints = configureLocationConstraints(appCtx);
+        locationConstraints = configureLocationConstraints(appCtx, scanFiles);
         configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_DELTA);
-        distributeFiles();
+        distributeFiles(scanFiles);
         issueWarnings(warnings, warningCollector);
     }
 
     private void issueWarnings(List<Warning> warnings, IWarningCollector warningCollector) {
-        if (!warnings.isEmpty() && warningCollector.shouldWarn()) {
+        if (!warnings.isEmpty()) {
             for (Warning warning : warnings) {
                 if (warningCollector.shouldWarn()) {
                     warningCollector.warn(warning);
@@ -151,7 +156,8 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         warnings.clear();
     }
 
-    private AlgebricksAbsolutePartitionConstraint configureLocationConstraints(ICcApplicationContext appCtx) {
+    private AlgebricksAbsolutePartitionConstraint configureLocationConstraints(ICcApplicationContext appCtx,
+            List<Row> scanFiles) {
         IClusterStateManager csm = appCtx.getClusterStateManager();
 
         String[] locations = csm.getClusterLocations().getLocations();
@@ -168,24 +174,30 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         return new AlgebricksAbsolutePartitionConstraint(locations);
     }
 
-    private void distributeFiles() {
-        final int numComputePartitions = getPartitionConstraint().getLocations().length;
-        schedule = new HashMap<>();
-        for (int i = 0; i < numComputePartitions; i++) {
-            schedule.put(i, new ArrayList<>());
+    private void distributeFiles(List<Row> scanFiles) {
+        final int partitionsCount = getPartitionConstraint().getLocations().length;
+        PriorityQueue<PartitionWorkLoadBasedOnSize> workloadQueue = new PriorityQueue<>(partitionsCount,
+                Comparator.comparingLong(PartitionWorkLoadBasedOnSize::getTotalSize));
+
+        // Prepare the workloads based on the number of partitions
+        for (int i = 0; i < partitionsCount; i++) {
+            workloadQueue.add(new PartitionWorkLoadBasedOnSize());
         }
-        int i = 0;
-        for (String scanFile : scanFiles) {
-            schedule.get(i).add(scanFile);
-            i = (i + 1) % numComputePartitions;
+        for (Row scanFileRow : scanFiles) {
+            PartitionWorkLoadBasedOnSize workload = workloadQueue.poll();
+            FileStatus fileStatus = InternalScanFileUtils.getAddFileStatus(scanFileRow);
+            workload.addScanFile(RowSerDe.serializeRowToJson(scanFileRow), fileStatus.getSize());
+            workloadQueue.add(workload);
         }
+        partitionWorkLoadsBasedOnSize.addAll(workloadQueue);
     }
 
     @Override
     public IRecordReader<?> createRecordReader(IExternalDataRuntimeContext context) throws HyracksDataException {
         try {
             int partition = context.getPartition();
-            return new DeltaFileRecordReader(schedule.get(partition), scanState, configuration, context);
+            return new DeltaFileRecordReader(partitionWorkLoadsBasedOnSize.get(partition).getScanFiles(), scanState,
+                    configuration, context);
         } catch (Exception e) {
             throw HyracksDataException.create(e);
         }
@@ -206,4 +218,31 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         return Collections.singleton(ExternalDataConstants.FORMAT_DELTA);
     }
 
+    public static class PartitionWorkLoadBasedOnSize implements Serializable {
+        private static final long serialVersionUID = 1L;
+        private final List<String> scanFiles = new ArrayList<>();
+        private long totalSize = 0;
+
+        public PartitionWorkLoadBasedOnSize() {
+        }
+
+        public List<String> getScanFiles() {
+            return scanFiles;
+        }
+
+        public void addScanFile(String scanFile, long size) {
+            this.scanFiles.add(scanFile);
+            this.totalSize += size;
+        }
+
+        public long getTotalSize() {
+            return totalSize;
+        }
+
+        @Override
+        public String toString() {
+            return "Files: " + scanFiles.size() + ", Total Size: " + totalSize;
+        }
+    }
+
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 50e72ed047..b3118702a1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -921,7 +921,7 @@ public class ExternalDataUtils {
     }
 
     public static boolean supportsPushdown(Map<String, String> properties) {
-        //Currently, only Apache Parquet format is supported
+        //Currently, only Apache Parquet/Delta table format is supported
         return isParquetFormat(properties) || isDeltaTable(properties);
     }
 


(asterixdb) 14/15: Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 7026faf391298e4e5227e6cf54b800d1c3132ba9
Merge: 5ddcc8f276 bdaf4e97e4
Author: Apache AsterixDB Gerrit <de...@asterixdb.apache.org>
AuthorDate: Mon Nov 25 00:44:07 2024 +0000

    Merge "Merge branch 'gerrit/trinity' into 'gerrit/goldfish'" into goldfish

 .../rules/PushAggregateIntoNestedSubplanRule.java     |  6 ++++--
 .../optimizer/rules/PushGroupByThroughProduct.java    |  4 +++-
 .../core/algebra/util/OperatorManipulationUtil.java   | 19 +++++++++++++++----
 .../rewriter/rules/AbstractDecorrelationRule.java     |  4 +++-
 .../rules/subplan/IntroduceGroupByForSubplanRule.java |  7 +++++--
 5 files changed, 30 insertions(+), 10 deletions(-)


(asterixdb) 02/15: [ASTERIXDB-3503][EXT] Add column filter for Delta Reader.

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit 6249ff58f37131cb5ab475f7fe2a98a930bac186
Author: Peeyush Gupta <pe...@couchbase.com>
AuthorDate: Mon Nov 18 17:36:24 2024 -0800

    [ASTERIXDB-3503][EXT] Add column filter for Delta Reader.
    
    - user model changes: no
    - storage format changes: no
    - interface changes: no
    
    Details:
    Reading only the required columns from the delta table.
    
    Ext-ref: MB-63840
    
    Change-Id: I809c692777349025d5ce0435c3a6068d432cd282
    Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19085
    Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
    Reviewed-by: Murtadha Hubail <mh...@apache.org>
    Tested-by: Murtadha Hubail <mh...@apache.org>
---
 .../aws/delta/AsterixTypeToDeltaTypeVisitor.java   | 145 +++++++++++++++++++++
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  |  39 +++++-
 .../aws/delta/converter/DeltaConverterContext.java |  10 +-
 .../asterix/external/parser/DeltaDataParser.java   |  13 +-
 .../asterix/external/util/ExternalDataUtils.java   |   2 +-
 5 files changed, 202 insertions(+), 7 deletions(-)

diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
new file mode 100644
index 0000000000..2c86132490
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AsterixTypeToDeltaTypeVisitor.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.aws.delta;
+
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.ALL_FIELDS_TYPE;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.EMPTY_TYPE;
+
+import java.util.Map;
+
+import org.apache.asterix.external.input.record.reader.aws.delta.converter.DeltaConverterContext;
+import org.apache.asterix.external.parser.DeltaDataParser;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.types.IATypeVisitor;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.Warning;
+
+import io.delta.kernel.types.ArrayType;
+import io.delta.kernel.types.DataType;
+import io.delta.kernel.types.StructType;
+
+/**
+ * This visitor clips the filesSchema stored in Delta table metadata using the expected type
+ */
+public class AsterixTypeToDeltaTypeVisitor implements IATypeVisitor<DataType, DataType> {
+
+    private final DeltaConverterContext context;
+    private Map<String, FunctionCallInformation> funcInfo;
+
+    public AsterixTypeToDeltaTypeVisitor(DeltaConverterContext context) {
+        this.context = context;
+    }
+
+    public StructType clipType(ARecordType rootType, StructType fileSchema,
+            Map<String, FunctionCallInformation> funcInfo) {
+        if (rootType == EMPTY_TYPE) {
+            return new StructType();
+        } else if (rootType == ALL_FIELDS_TYPE) {
+            return fileSchema;
+        }
+        StructType builder = new StructType();
+        this.funcInfo = funcInfo;
+        return clipObjectChildren(builder, rootType, fileSchema);
+    }
+
+    @Override
+    public DataType visit(ARecordType recordType, DataType arg) {
+        if (isNotCompatibleType(arg, recordType)) {
+            return null;
+        }
+        StructType builder = new StructType();
+        builder = clipObjectChildren(builder, recordType, (StructType) arg);
+        if (builder.fields().size() == 0) {
+            return null;
+        }
+        return builder;
+    }
+
+    @Override
+    public DataType visit(AbstractCollectionType collectionType, DataType arg) {
+        if (isNotCompatibleType(arg, collectionType)) {
+            return null;
+        }
+        DataType elementSchema = ((ArrayType) arg).getElementType();
+        DataType requestedChildType = collectionType.getItemType().accept(this, elementSchema);
+        return new ArrayType(requestedChildType, true);
+    }
+
+    private StructType clipObjectChildren(StructType builder, ARecordType recordType, StructType arg) {
+        String[] fieldNames = recordType.getFieldNames();
+        IAType[] fieldTypes = recordType.getFieldTypes();
+        for (int i = 0; i < fieldNames.length; i++) {
+            // If the field is not present in the file schema, we skip it
+            if (arg.fieldNames().contains(fieldNames[i])) {
+                DataType type = arg.get(fieldNames[i]).getDataType();
+                DataType childType = fieldTypes[i].accept(this, type);
+                if (childType != null) {
+                    // We only add non-MISSING children
+                    builder = builder.add(fieldNames[i], childType);
+                }
+            }
+        }
+        return builder;
+    }
+
+    @Override
+    public DataType visit(AUnionType unionType, DataType arg) {
+        if (arg instanceof ArrayType) {
+            return unionType.getType(ATypeTag.ARRAY).accept(this, arg);
+        } else {
+            return unionType.getType(ATypeTag.OBJECT).accept(this, arg);
+        }
+    }
+
+    @Override
+    public DataType visitFlat(IAType node, DataType arg) {
+        return arg;
+    }
+
+    private boolean isNotCompatibleType(DataType type, IAType node) {
+        // typeName is unique
+        FunctionCallInformation info = funcInfo.get(node.getTypeName());
+        ATypeTag actualType = null;
+        try {
+            actualType = DeltaDataParser.getTypeTag(type, false, context);
+        } catch (HyracksDataException e) {
+            throw new RuntimeException(e);
+        }
+        ATypeTag expectedType = node.getTypeTag();
+
+        boolean isNotExpected = actualType != expectedType;
+        if (isNotExpected) {
+            //If no warning is created, then it means it has been reported
+            Warning warning = null;
+            if (actualType != ATypeTag.SYSTEM_NULL) {
+                warning = info.createWarning(expectedType, actualType);
+            }
+            if (warning != null) {
+                //New warning that we saw for the first time. We should report it.
+                context.getWarnings().add(warning);
+            }
+        }
+        return isNotExpected;
+    }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
index 9909cc32bd..5ce2c78152 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/AwsS3DeltaReaderFactory.java
@@ -20,6 +20,7 @@ package org.apache.asterix.external.input.record.reader.aws.delta;
 
 import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -34,8 +35,12 @@ import org.apache.asterix.common.external.IExternalFilterEvaluatorFactory;
 import org.apache.asterix.external.api.IExternalDataRuntimeContext;
 import org.apache.asterix.external.api.IRecordReader;
 import org.apache.asterix.external.api.IRecordReaderFactory;
+import org.apache.asterix.external.input.record.reader.aws.delta.converter.DeltaConverterContext;
 import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.HDFSUtils;
 import org.apache.asterix.external.util.aws.s3.S3Constants;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.runtime.projection.FunctionCallInformation;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
@@ -43,6 +48,7 @@ import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.api.application.IServiceContext;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
 import org.apache.hyracks.api.exceptions.IWarningCollector;
+import org.apache.hyracks.api.exceptions.Warning;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -52,6 +58,7 @@ import io.delta.kernel.data.FilteredColumnarBatch;
 import io.delta.kernel.data.Row;
 import io.delta.kernel.defaults.engine.DefaultEngine;
 import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructType;
 import io.delta.kernel.utils.CloseableIterator;
 
 public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
@@ -87,6 +94,10 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         if (serviceEndpoint != null) {
             conf.set(S3Constants.HADOOP_SERVICE_END_POINT, serviceEndpoint);
         }
+        conf.set(ExternalDataConstants.KEY_REQUESTED_FIELDS,
+                configuration.getOrDefault(ExternalDataConstants.KEY_REQUESTED_FIELDS, ""));
+        conf.set(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION,
+                configuration.getOrDefault(ExternalDataConstants.KEY_HADOOP_ASTERIX_FUNCTION_CALL_INFORMATION, ""));
         String tableMetadataPath = S3Constants.HADOOP_S3_PROTOCOL + "://"
                 + configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME) + '/'
                 + configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
@@ -96,7 +107,21 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         Engine engine = DefaultEngine.create(conf);
         io.delta.kernel.Table table = io.delta.kernel.Table.forPath(engine, tableMetadataPath);
         Snapshot snapshot = table.getLatestSnapshot(engine);
-        Scan scan = snapshot.getScanBuilder(engine).withReadSchema(engine, snapshot.getSchema(engine)).build();
+
+        List<Warning> warnings = new ArrayList<>();
+        DeltaConverterContext converterContext = new DeltaConverterContext(configuration, warnings);
+        AsterixTypeToDeltaTypeVisitor visitor = new AsterixTypeToDeltaTypeVisitor(converterContext);
+        StructType requiredSchema;
+        try {
+            ARecordType expectedType = HDFSUtils.getExpectedType(conf);
+            Map<String, FunctionCallInformation> functionCallInformationMap =
+                    HDFSUtils.getFunctionCallInformationMap(conf);
+            StructType fileSchema = snapshot.getSchema(engine);
+            requiredSchema = visitor.clipType(expectedType, fileSchema, functionCallInformationMap);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        Scan scan = snapshot.getScanBuilder(engine).withReadSchema(engine, requiredSchema).build();
         scanState = RowSerDe.serializeRowToJson(scan.getScanState(engine));
         CloseableIterator<FilteredColumnarBatch> iter = scan.getScanFiles(engine);
 
@@ -112,6 +137,18 @@ public class AwsS3DeltaReaderFactory implements IRecordReaderFactory<Object> {
         locationConstraints = configureLocationConstraints(appCtx);
         configuration.put(ExternalDataConstants.KEY_PARSER, ExternalDataConstants.FORMAT_DELTA);
         distributeFiles();
+        issueWarnings(warnings, warningCollector);
+    }
+
+    private void issueWarnings(List<Warning> warnings, IWarningCollector warningCollector) {
+        if (!warnings.isEmpty() && warningCollector.shouldWarn()) {
+            for (Warning warning : warnings) {
+                if (warningCollector.shouldWarn()) {
+                    warningCollector.warn(warning);
+                }
+            }
+        }
+        warnings.clear();
     }
 
     private AlgebricksAbsolutePartitionConstraint configureLocationConstraints(ICcApplicationContext appCtx) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/converter/DeltaConverterContext.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/converter/DeltaConverterContext.java
index 81e465c100..a404e2fbc6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/converter/DeltaConverterContext.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/delta/converter/DeltaConverterContext.java
@@ -19,6 +19,7 @@
 package org.apache.asterix.external.input.record.reader.aws.delta.converter;
 
 import java.io.DataOutput;
+import java.util.List;
 import java.util.Map;
 import java.util.TimeZone;
 
@@ -32,6 +33,7 @@ import org.apache.asterix.om.base.AMutableDateTime;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.Warning;
 
 public class DeltaConverterContext extends ParserContext {
     @SuppressWarnings("unchecked")
@@ -47,8 +49,10 @@ public class DeltaConverterContext extends ParserContext {
     private final int timeZoneOffset;
     private final AMutableDate mutableDate = new AMutableDate(0);
     private final AMutableDateTime mutableDateTime = new AMutableDateTime(0);
+    private final List<Warning> warnings;
 
-    public DeltaConverterContext(Map<String, String> configuration) {
+    public DeltaConverterContext(Map<String, String> configuration, List<Warning> warnings) {
+        this.warnings = warnings;
         decimalToDouble = Boolean.parseBoolean(configuration
                 .getOrDefault(ExternalDataConstants.DeltaOptions.DECIMAL_TO_DOUBLE, ExternalDataConstants.FALSE));
         timestampAsLong = Boolean.parseBoolean(configuration
@@ -96,4 +100,8 @@ public class DeltaConverterContext extends ParserContext {
     public boolean isDateAsInt() {
         return dateAsInt;
     }
+
+    public List<Warning> getWarnings() {
+        return warnings;
+    }
 }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DeltaDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DeltaDataParser.java
index ea02d77acb..adb846d270 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DeltaDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/DeltaDataParser.java
@@ -23,6 +23,8 @@ import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
 import java.io.DataOutput;
 import java.io.IOException;
 import java.math.BigDecimal;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
@@ -42,6 +44,7 @@ import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
 import org.apache.asterix.om.types.ATypeTag;
 import org.apache.avro.AvroRuntimeException;
 import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.Warning;
 import org.apache.hyracks.data.std.api.IMutableValueStorage;
 import org.apache.hyracks.data.std.api.IValueReference;
 
@@ -68,7 +71,8 @@ public class DeltaDataParser extends AbstractDataParser implements IRecordDataPa
     private final IExternalFilterValueEmbedder valueEmbedder;
 
     public DeltaDataParser(IExternalDataRuntimeContext context, Map<String, String> conf) {
-        parserContext = new DeltaConverterContext(conf);
+        List<Warning> warnings = new ArrayList<>();
+        parserContext = new DeltaConverterContext(conf, warnings);
         valueEmbedder = context.getValueEmbedder();
     }
 
@@ -91,7 +95,7 @@ public class DeltaDataParser extends AbstractDataParser implements IRecordDataPa
         for (int i = 0; i < schema.fields().size(); i++) {
             DataType fieldSchema = schema.fields().get(i).getDataType();
             String fieldName = schema.fieldNames().get(i);
-            ATypeTag typeTag = getTypeTag(fieldSchema, record.isNullAt(i));
+            ATypeTag typeTag = getTypeTag(fieldSchema, record.isNullAt(i), parserContext);
             IValueReference value = null;
             if (valueEmbedder.shouldEmbed(fieldName, typeTag)) {
                 value = valueEmbedder.getEmbeddedValue();
@@ -120,7 +124,7 @@ public class DeltaDataParser extends AbstractDataParser implements IRecordDataPa
         for (int i = 0; i < schema.fields().size(); i++) {
             DataType fieldSchema = schema.fields().get(i).getDataType();
             String fieldName = schema.fieldNames().get(i);
-            ATypeTag typeTag = getTypeTag(fieldSchema, column.getChild(i).isNullAt(index));
+            ATypeTag typeTag = getTypeTag(fieldSchema, column.getChild(i).isNullAt(index), parserContext);
             IValueReference value = null;
             if (valueEmbedder.shouldEmbed(fieldName, typeTag)) {
                 value = valueEmbedder.getEmbeddedValue();
@@ -157,7 +161,8 @@ public class DeltaDataParser extends AbstractDataParser implements IRecordDataPa
         parserContext.exitCollection(valueBuffer, arrayBuilder);
     }
 
-    private ATypeTag getTypeTag(DataType schema, boolean isNull) throws HyracksDataException {
+    public static ATypeTag getTypeTag(DataType schema, boolean isNull, DeltaConverterContext parserContext)
+            throws HyracksDataException {
         if (isNull) {
             return ATypeTag.NULL;
         }
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index bc5b8c3326..50e72ed047 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -922,7 +922,7 @@ public class ExternalDataUtils {
 
     public static boolean supportsPushdown(Map<String, String> properties) {
         //Currently, only Apache Parquet format is supported
-        return isParquetFormat(properties);
+        return isParquetFormat(properties) || isDeltaTable(properties);
     }
 
     /**


(asterixdb) 15/15: Merge branch 'gerrit/goldfish' into 'master'

Posted by mb...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

mblow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git

commit dec9f981e425ff749d74df9e08120c71c867a7ba
Merge: d21a38bf55 7026faf391
Author: Michael Blow <mi...@couchbase.com>
AuthorDate: Mon Nov 25 07:51:43 2024 -0500

    Merge branch 'gerrit/goldfish' into 'master'
    
    Change-Id: Ia6f1478cc62fb07c2eba3c6c9828b568c43a3cc0

 .../rules/PushAggregateIntoNestedSubplanRule.java  |   6 +-
 .../optimizer/rules/PushGroupByThroughProduct.java |   4 +-
 .../external_dataset/ExternalDatasetTestUtils.java |   4 +
 .../deltalake/DeltaAllTypeGenerator.java           | 268 +++++++++++++++++++++
 .../deltalake-all-type.00.ddl.sqlpp                |  52 ++--
 .../deltalake-all-type.01.query.sqlpp              |  30 +--
 .../deltalake-all-type.02.query.sqlpp              |  30 +--
 .../deltalake-field-access-pushdown.00.ddl.sqlpp   |  53 ++--
 .../deltalake-field-access-pushdown.01.query.sqlpp |  30 +--
 .../deltalake-field-access-pushdown.02.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.03.query.sqlpp |  32 +--
 .../deltalake-field-access-pushdown.04.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.05.query.sqlpp |  32 +--
 .../deltalake-field-access-pushdown.06.query.sqlpp |  34 +--
 .../deltalake-field-access-pushdown.07.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.08.query.sqlpp |  34 +--
 .../deltalake-field-access-pushdown.09.query.sqlpp |  33 +--
 .../deltalake-field-access-pushdown.10.query.sqlpp |  34 +--
 .../deltalake-table-not-exists.00.ddl.sqlpp        |  37 +--
 .../deltalake-all-type/deltalake-all-type.01.adm   |   5 +
 .../deltalake-all-type/deltalake-all-type.02.adm   |   5 +
 .../deltalake-field-access-pushdown.01.adm         |   5 +
 .../deltalake-field-access-pushdown.02.plan        |  22 ++
 .../deltalake-field-access-pushdown.03.adm         |   5 +
 .../deltalake-field-access-pushdown.04.plan        |  22 ++
 .../deltalake-field-access-pushdown.05.adm         |   5 +
 .../deltalake-field-access-pushdown.06.plan        |  50 ++++
 .../deltalake-field-access-pushdown.07.adm         |   5 +
 .../deltalake-field-access-pushdown.08.plan        |  50 ++++
 .../deltalake-field-access-pushdown.09.adm         |   5 +
 .../deltalake-field-access-pushdown.10.adm         |  46 ++++
 .../runtimets/testsuite_external_dataset_s3.xml    |  19 ++
 .../NoOpFrameOperationCallbackFactory.java         |   5 +
 .../aws/delta/AsterixDeltaRuntimeException.java    |  34 +--
 .../aws/delta/AsterixTypeToDeltaTypeVisitor.java   | 145 +++++++++++
 .../reader/aws/delta/AwsS3DeltaReaderFactory.java  | 150 +++++++++---
 .../reader/aws/delta/DeltaFileRecordReader.java    |  10 +-
 .../aws/delta/converter/DeltaConverterContext.java |  10 +-
 .../asterix/external/parser/DeltaDataParser.java   |  13 +-
 .../asterix/external/util/ExternalDataUtils.java   |  34 ++-
 .../asterix/external/util/aws/s3/S3AuthUtils.java  |   4 +
 .../external/util/google/gcs/GCSConstants.java     |  12 +
 .../asterix/external/util/google/gcs/GCSUtils.java |  35 ++-
 .../LSMPrimaryInsertOperatorNodePushable.java      |  10 +-
 .../LSMPrimaryUpsertOperatorNodePushable.java      |  16 +-
 .../runtime/operators/StandardBatchController.java |  45 ++++
 .../algebra/util/OperatorManipulationUtil.java     |  19 +-
 .../rewriter/rules/AbstractDecorrelationRule.java  |   4 +-
 .../subplan/IntroduceGroupByForSubplanRule.java    |   7 +-
 .../apache/hyracks/api/comm/IFrameAppender.java    |   2 +-
 .../hyracks/api/util/HyracksThrowingAction.java}   |  30 +--
 .../org/apache/hyracks/api/util/InvokeUtil.java    |  36 +++
 .../common/io/MessagingFrameTupleAppender.java     |   9 -
 .../hyracks/dataflow/common/utils/TaskUtil.java    |  14 ++
 ...perationCallback.java => IBatchController.java} |  29 +--
 .../am/lsm/common/api/IFrameOperationCallback.java |  14 +-
 .../storage/am/lsm/common/api/ILSMHarness.java     |   9 +-
 .../storage/am/lsm/common/impls/LSMHarness.java    |  31 ++-
 .../am/lsm/common/impls/LSMTreeIndexAccessor.java  |   6 +-
 59 files changed, 1232 insertions(+), 557 deletions(-)

diff --cc asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
index 2ba1844a86,bf0938b961..f36d25dd5f
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3AuthUtils.java
@@@ -468,5 -346,241 +469,8 @@@ public class S3AuthUtils 
          if (!response.sdkHttpResponse().isSuccessful()) {
              throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_CONTAINER_NOT_FOUND, container);
          }
+         if (isDeltaTable(configuration)) {
+             validateDeltaTableExists(configuration);
+         }
      }
 -
 -    /**
 -     * Checks for a single object in the specified bucket to determine if the bucket is empty or not.
 -     *
 -     * @param s3Client  s3 client
 -     * @param container the container name
 -     * @param prefix    Prefix to be used
 -     * @param useOldApi flag whether to use the old API or not
 -     * @return returns the S3 response
 -     */
 -    private static S3Response isBucketEmpty(S3Client s3Client, String container, String prefix, boolean useOldApi) {
 -        S3Response response;
 -        if (useOldApi) {
 -            ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder();
 -            listObjectsBuilder.prefix(prefix);
 -            response = s3Client.listObjects(listObjectsBuilder.bucket(container).maxKeys(1).build());
 -        } else {
 -            ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder();
 -            listObjectsBuilder.prefix(prefix);
 -            response = s3Client.listObjectsV2(listObjectsBuilder.bucket(container).maxKeys(1).build());
 -        }
 -        return response;
 -    }
 -
 -    /**
 -     * Returns the lists of S3 objects.
 -     *
 -     * @param configuration         properties
 -     * @param includeExcludeMatcher include/exclude matchers to apply
 -     */
 -    public static List<S3Object> listS3Objects(Map<String, String> configuration,
 -            AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
 -            IWarningCollector warningCollector, ExternalDataPrefix externalDataPrefix,
 -            IExternalFilterEvaluator evaluator) throws CompilationException, HyracksDataException {
 -        // Prepare to retrieve the objects
 -        List<S3Object> filesOnly;
 -        String container = configuration.get(ExternalDataConstants.CONTAINER_NAME_FIELD_NAME);
 -        S3Client s3Client = buildAwsS3Client(configuration);
 -        String prefix = getPrefix(configuration);
 -
 -        try {
 -            filesOnly = listS3Objects(s3Client, container, prefix, includeExcludeMatcher, externalDataPrefix, evaluator,
 -                    warningCollector);
 -        } catch (S3Exception ex) {
 -            // New API is not implemented, try falling back to old API
 -            try {
 -                // For error code, see https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 -                if (ex.awsErrorDetails().errorCode().equals(ERROR_METHOD_NOT_IMPLEMENTED)) {
 -                    filesOnly = oldApiListS3Objects(s3Client, container, prefix, includeExcludeMatcher,
 -                            externalDataPrefix, evaluator, warningCollector);
 -                } else {
 -                    throw ex;
 -                }
 -            } catch (SdkException ex2) {
 -                throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex2, getMessageOrToString(ex));
 -            }
 -        } catch (SdkException ex) {
 -            throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex, getMessageOrToString(ex));
 -        } finally {
 -            if (s3Client != null) {
 -                CleanupUtils.close(s3Client, null);
 -            }
 -        }
 -
 -        // Warn if no files are returned
 -        if (filesOnly.isEmpty() && warningCollector.shouldWarn()) {
 -            Warning warning = Warning.of(null, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
 -            warningCollector.warn(warning);
 -        }
 -
 -        return filesOnly;
 -    }
 -
 -    /**
 -     * Uses the latest API to retrieve the objects from the storage.
 -     *
 -     * @param s3Client              S3 client
 -     * @param container             container name
 -     * @param prefix                definition prefix
 -     * @param includeExcludeMatcher include/exclude matchers to apply
 -     */
 -    private static List<S3Object> listS3Objects(S3Client s3Client, String container, String prefix,
 -            AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
 -            ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
 -            IWarningCollector warningCollector) throws HyracksDataException {
 -        String newMarker = null;
 -        List<S3Object> filesOnly = new ArrayList<>();
 -
 -        ListObjectsV2Response listObjectsResponse;
 -        ListObjectsV2Request.Builder listObjectsBuilder = ListObjectsV2Request.builder().bucket(container);
 -        listObjectsBuilder.prefix(prefix);
 -
 -        while (true) {
 -            // List the objects from the start, or from the last marker in case of truncated result
 -            if (newMarker == null) {
 -                listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.build());
 -            } else {
 -                listObjectsResponse = s3Client.listObjectsV2(listObjectsBuilder.continuationToken(newMarker).build());
 -            }
 -
 -            // Collect the paths to files only
 -            collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
 -                    includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator,
 -                    warningCollector);
 -
 -            // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
 -            if (listObjectsResponse.isTruncated() != null && listObjectsResponse.isTruncated()) {
 -                newMarker = listObjectsResponse.nextContinuationToken();
 -            } else {
 -                break;
 -            }
 -        }
 -
 -        return filesOnly;
 -    }
 -
 -    /**
 -     * Uses the old API (in case the new API is not implemented) to retrieve the objects from the storage
 -     *
 -     * @param s3Client              S3 client
 -     * @param container             container name
 -     * @param prefix                definition prefix
 -     * @param includeExcludeMatcher include/exclude matchers to apply
 -     */
 -    private static List<S3Object> oldApiListS3Objects(S3Client s3Client, String container, String prefix,
 -            AbstractExternalInputStreamFactory.IncludeExcludeMatcher includeExcludeMatcher,
 -            ExternalDataPrefix externalDataPrefix, IExternalFilterEvaluator evaluator,
 -            IWarningCollector warningCollector) throws HyracksDataException {
 -        String newMarker = null;
 -        List<S3Object> filesOnly = new ArrayList<>();
 -
 -        ListObjectsResponse listObjectsResponse;
 -        ListObjectsRequest.Builder listObjectsBuilder = ListObjectsRequest.builder().bucket(container);
 -        listObjectsBuilder.prefix(prefix);
 -
 -        while (true) {
 -            // List the objects from the start, or from the last marker in case of truncated result
 -            if (newMarker == null) {
 -                listObjectsResponse = s3Client.listObjects(listObjectsBuilder.build());
 -            } else {
 -                listObjectsResponse = s3Client.listObjects(listObjectsBuilder.marker(newMarker).build());
 -            }
 -
 -            // Collect the paths to files only
 -            collectAndFilterFiles(listObjectsResponse.contents(), includeExcludeMatcher.getPredicate(),
 -                    includeExcludeMatcher.getMatchersList(), filesOnly, externalDataPrefix, evaluator,
 -                    warningCollector);
 -
 -            // Mark the flag as done if done, otherwise, get the marker of the previous response for the next request
 -            if (listObjectsResponse.isTruncated() != null && listObjectsResponse.isTruncated()) {
 -                newMarker = listObjectsResponse.nextMarker();
 -            } else {
 -                break;
 -            }
 -        }
 -
 -        return filesOnly;
 -    }
 -
 -    /**
 -     * Collects only files that pass all tests
 -     *
 -     * @param s3Objects          s3 objects
 -     * @param predicate          predicate
 -     * @param matchers           matchers
 -     * @param filesOnly          filtered files
 -     * @param externalDataPrefix external data prefix
 -     * @param evaluator          evaluator
 -     */
 -    private static void collectAndFilterFiles(List<S3Object> s3Objects, BiPredicate<List<Matcher>, String> predicate,
 -            List<Matcher> matchers, List<S3Object> filesOnly, ExternalDataPrefix externalDataPrefix,
 -            IExternalFilterEvaluator evaluator, IWarningCollector warningCollector) throws HyracksDataException {
 -        for (S3Object object : s3Objects) {
 -            if (ExternalDataUtils.evaluate(object.key(), predicate, matchers, externalDataPrefix, evaluator,
 -                    warningCollector)) {
 -                filesOnly.add(object);
 -            }
 -        }
 -    }
 -
 -    public static Map<String, List<String>> S3ObjectsOfSingleDepth(Map<String, String> configuration, String container,
 -            String prefix) throws CompilationException, HyracksDataException {
 -        // create s3 client
 -        S3Client s3Client = buildAwsS3Client(configuration);
 -        // fetch all the s3 objects
 -        return listS3ObjectsOfSingleDepth(s3Client, container, prefix);
 -    }
 -
 -    /**
 -     * Uses the latest API to retrieve the objects from the storage of a single level.
 -     *
 -     * @param s3Client              S3 client
 -     * @param container             container name
 -     * @param prefix                definition prefix
 -     */
 -    private static Map<String, List<String>> listS3ObjectsOfSingleDepth(S3Client s3Client, String container,
 -            String prefix) throws HyracksDataException {
 -        Map<String, List<String>> allObjects = new HashMap<>();
 -        ListObjectsV2Iterable listObjectsInterable;
 -        ListObjectsV2Request.Builder listObjectsBuilder =
 -                ListObjectsV2Request.builder().bucket(container).prefix(prefix).delimiter("/");
 -        listObjectsBuilder.prefix(prefix);
 -        List<String> files = new ArrayList<>();
 -        List<String> folders = new ArrayList<>();
 -        // to skip the prefix as a file from the response
 -        boolean checkPrefixInFile = true;
 -        listObjectsInterable = s3Client.listObjectsV2Paginator(listObjectsBuilder.build());
 -        for (ListObjectsV2Response response : listObjectsInterable) {
 -            // put all the files
 -            for (S3Object object : response.contents()) {
 -                String fileName = object.key();
 -                fileName = fileName.substring(prefix.length(), fileName.length());
 -                if (checkPrefixInFile) {
 -                    if (prefix.equals(object.key()))
 -                        checkPrefixInFile = false;
 -                    else {
 -                        files.add(fileName);
 -                    }
 -                } else {
 -                    files.add(fileName);
 -                }
 -            }
 -            // put all the folders
 -            for (CommonPrefix object : response.commonPrefixes()) {
 -                String folderName = object.prefix();
 -                folderName = folderName.substring(prefix.length(), folderName.length());
 -                folders.add(folderName.endsWith("/") ? folderName.substring(0, folderName.length() - 1) : folderName);
 -            }
 -        }
 -        allObjects.put("files", files);
 -        allObjects.put("folders", folders);
 -        return allObjects;
 -    }
  }