(iceberg) branch main updated: Core: Optimize manifest evaluation for super wide tables (#9147)

[fo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 197b61e756 Core: Optimize manifest evaluation for super wide tables (#9147)
197b61e756 is described below

commit 197b61e756155b95f6c889e7718bf1c2af70af7f
Author: Irshad CC <ir...@gmail.com>
AuthorDate: Mon Dec 25 19:52:48 2023 +0800

    Core: Optimize manifest evaluation for super wide tables (#9147)
---
 .../org/apache/iceberg/expressions/NamedReference.java   |  2 +-
 api/src/main/java/org/apache/iceberg/types/Types.java    | 16 ++++++++++++++++
 2 files changed, 17 insertions(+), 1 deletion(-)

diff --git a/api/src/main/java/org/apache/iceberg/expressions/NamedReference.java b/api/src/main/java/org/apache/iceberg/expressions/NamedReference.java
index 32d1a17e28..cc5ba3ceaf 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/NamedReference.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/NamedReference.java
@@ -38,7 +38,7 @@ public class NamedReference<T> implements UnboundTerm<T>, Reference<T> {
 
   @Override
   public BoundReference<T> bind(Types.StructType struct, boolean caseSensitive) {
-    Schema schema = new Schema(struct.fields());
+    Schema schema = struct.asSchema();
     Types.NestedField field =
         caseSensitive ? schema.findField(name) : schema.caseInsensitiveFindField(name);
 
diff --git a/api/src/main/java/org/apache/iceberg/types/Types.java b/api/src/main/java/org/apache/iceberg/types/Types.java
index da70dd9ac6..dda842c9e1 100644
--- a/api/src/main/java/org/apache/iceberg/types/Types.java
+++ b/api/src/main/java/org/apache/iceberg/types/Types.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import org.apache.iceberg.Schema;
 import org.apache.iceberg.relocated.com.google.common.base.Joiner;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -537,6 +538,7 @@ public class Types {
     private final NestedField[] fields;
 
     // lazy values
+    private transient Schema schema = null;
     private transient List<NestedField> fieldList = null;
     private transient Map<String, NestedField> fieldsByName = null;
     private transient Map<String, NestedField> fieldsByLowerCaseName = null;
@@ -592,6 +594,20 @@ public class Types {
       return this;
     }
 
+    /**
+     * Returns a schema which contains the columns inside struct type. This method can be used to
+     * avoid expensive conversion of StructType containing large number of columns to Schema during
+     * manifest evaluation.
+     *
+     * @return the schema containing columns of struct type.
+     */
+    public Schema asSchema() {
+      if (this.schema == null) {
+        this.schema = new Schema(Arrays.asList(this.fields));
+      }
+      return this.schema;
+    }
+
     @Override
     public String toString() {
       return String.format("struct<%s>", FIELD_SEP.join(fields));