You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by "kinolaev (via GitHub)" <gi...@apache.org> on 2026/08/28 18:54:45 UTC

[PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

kinolaev opened a new pull request, #17864:
URL: https://github.com/apache/iceberg/pull/17864

   `CharSequenceMap.computeIfAbsent` computes a hash code for the provided key on every call. Because rows in a position delete file are sorted by `file_path`, we only need to call `computeIfAbsent` when the key changes.
   
   ```
   Benchmark                                      (numDataFiles)  (numDeletes)  Mode  Cnt  Score    Error  Units
   ToPositionIndexesBenchmark.computeIfAbsent                  1       1000000    ss    5  0.310 ±  0.002   s/op
   ToPositionIndexesBenchmark.computeIfAbsent                 10       1000000    ss    5  0.314 ±  0.012   s/op
   ToPositionIndexesBenchmark.computeIfAbsent                100       1000000    ss    5  0.311 ±  0.009   s/op
   ToPositionIndexesBenchmark.computeIfAbsent               1000       1000000    ss    5  0.133 ±  0.008   s/op
   ToPositionIndexesBenchmark.computeIfAbsent              10000       1000000    ss    5  0.316 ±  0.028   s/op
   ToPositionIndexesBenchmark.computeIfAbsent             100000       1000000    ss    5  0.196 ±  0.050   s/op
   ToPositionIndexesBenchmark.computeIfAbsent            1000000       1000000    ss    5  0.748 ±  0.578   s/op
   ToPositionIndexesBenchmark.stringEquals                     1       1000000    ss    5  0.023 ±  0.050   s/op
   ToPositionIndexesBenchmark.stringEquals                    10       1000000    ss    5  0.016 ±  0.001   s/op
   ToPositionIndexesBenchmark.stringEquals                   100       1000000    ss    5  0.017 ±  0.001   s/op
   ToPositionIndexesBenchmark.stringEquals                  1000       1000000    ss    5  0.019 ±  0.011   s/op
   ToPositionIndexesBenchmark.stringEquals                 10000       1000000    ss    5  0.027 ±  0.025   s/op
   ToPositionIndexesBenchmark.stringEquals                100000       1000000    ss    5  0.068 ±  0.027   s/op
   ToPositionIndexesBenchmark.stringEquals               1000000       1000000    ss    5  0.622 ±  0.519   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath              1       1000000    ss    5  0.151 ±  0.007   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath             10       1000000    ss    5  0.151 ±  0.005   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath            100       1000000    ss    5  0.153 ±  0.001   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath           1000       1000000    ss    5  0.153 ±  0.004   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath          10000       1000000    ss    5  0.277 ±  0.012   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath         100000       1000000    ss    5  0.193 ±  0.017   s/op
   ToPositionIndexesBenchmark.comparatorsFilePath        1000000       1000000    ss    5  0.612 ±  0.378   s/op
   ```
   
   <details>
   
   <summary>ToPositionIndexesBenchmark.java</summary>
   
   ```java
   package org.apache.iceberg.deletes;
   
   import java.io.IOException;
   import java.io.UncheckedIOException;
   import java.util.List;
   import java.util.Locale;
   import java.util.concurrent.TimeUnit;
   import org.apache.iceberg.Accessor;
   import org.apache.iceberg.DeleteFile;
   import org.apache.iceberg.MetadataColumns;
   import org.apache.iceberg.Schema;
   import org.apache.iceberg.StructLike;
   import org.apache.iceberg.io.CloseableIterable;
   import org.apache.iceberg.io.DeleteSchemaUtil;
   import org.apache.iceberg.relocated.com.google.common.collect.Lists;
   import org.apache.iceberg.types.Comparators;
   import org.apache.iceberg.util.CharSequenceMap;
   import org.openjdk.jmh.annotations.Benchmark;
   import org.openjdk.jmh.annotations.BenchmarkMode;
   import org.openjdk.jmh.annotations.Fork;
   import org.openjdk.jmh.annotations.Measurement;
   import org.openjdk.jmh.annotations.Mode;
   import org.openjdk.jmh.annotations.Param;
   import org.openjdk.jmh.annotations.Scope;
   import org.openjdk.jmh.annotations.Setup;
   import org.openjdk.jmh.annotations.State;
   import org.openjdk.jmh.annotations.Threads;
   import org.openjdk.jmh.annotations.Timeout;
   import org.openjdk.jmh.annotations.Warmup;
   import org.openjdk.jmh.infra.Blackhole;
   
   @Fork(1)
   @State(Scope.Benchmark)
   @Warmup(iterations = 3)
   @Measurement(iterations = 5)
   @BenchmarkMode(Mode.SingleShotTime)
   @Timeout(time = 10, timeUnit = TimeUnit.MINUTES)
   public class ToPositionIndexesBenchmark {
     private static final Schema POS_DELETE_SCHEMA = DeleteSchemaUtil.pathPosSchema();
     private static final Accessor<StructLike> FILENAME_ACCESSOR =
         POS_DELETE_SCHEMA.accessorForField(MetadataColumns.DELETE_FILE_PATH.fieldId());
     private static final Accessor<StructLike> POSITION_ACCESSOR =
         POS_DELETE_SCHEMA.accessorForField(MetadataColumns.DELETE_FILE_POS.fieldId());
   
     @Param("1000000")
     private int numDeletes;
   
     @Param({"1", "10", "100", "1000", "10000", "100000", "1000000"})
     private int numDataFiles;
   
     private List<StructLike> deletes;
     private String pathTemplate =
         "s3://pretty-long-bucket-name/pretty-long-namespace-name/pretty-long-table-name/data/%07d-data.parquet";
   
     @Setup
     public void setupBenchmark() {
       deletes = Lists.newArrayListWithExpectedSize(numDeletes);
       int numDeletesPerDataFile = numDeletes / numDataFiles;
       for (int index = 0; index < numDeletes; index++) {
         String path = String.format(Locale.ROOT, pathTemplate, index / numDeletesPerDataFile);
         deletes.add(new PositionDeleteRow(path, index % numDeletesPerDataFile));
       }
     }
   
     @Benchmark
     @Threads(1)
     public void computeIfAbsent(Blackhole blackhole) {
       blackhole.consume(toPositionIndexesComputeIfAbsent(CloseableIterable.withNoopClose(deletes), null));
     }
   
     // https://github.com/apache/iceberg/blob/apache-iceberg-1.11.0/core/src/main/java/org/apache/iceberg/deletes/Deletes.java#L139
     private static <T extends StructLike> CharSequenceMap<PositionDeleteIndex> toPositionIndexesComputeIfAbsent(
         CloseableIterable<T> posDeletes, DeleteFile file) {
       CharSequenceMap<PositionDeleteIndex> indexes = CharSequenceMap.create();
   
       try (CloseableIterable<T> deletes = posDeletes) {
         for (T delete : deletes) {
           CharSequence filePath = (CharSequence) FILENAME_ACCESSOR.get(delete);
           long position = (long) POSITION_ACCESSOR.get(delete);
           PositionDeleteIndex index =
               indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));
           index.delete(position);
         }
       } catch (IOException e) {
         throw new UncheckedIOException("Failed to close position delete source", e);
       }
   
       return indexes;
     }
     @Benchmark
     @Threads(1)
     public void comparatorsFilePath(Blackhole blackhole) {
       blackhole.consume(
           toPositionIndexesComparatorsFilePath(CloseableIterable.withNoopClose(deletes), null));
     }
   
     public static <T extends StructLike>
         CharSequenceMap<PositionDeleteIndex> toPositionIndexesComparatorsFilePath(
             CloseableIterable<T> posDeletes, DeleteFile file) {
       CharSequenceMap<PositionDeleteIndex> indexes = CharSequenceMap.create();
   
       try (CloseableIterable<T> deletes = posDeletes) {
         CharSequence lastFilePath = null;
         PositionDeleteIndex index = null;
         for (T delete : deletes) {
           CharSequence filePath = (CharSequence) FILENAME_ACCESSOR.get(delete);
           long position = (long) POSITION_ACCESSOR.get(delete);
           if (lastFilePath == null || Comparators.filePath().compare(lastFilePath, filePath) != 0) {
             lastFilePath = filePath;
             index = indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));
           }
           index.delete(position);
         }
       } catch (IOException e) {
         throw new UncheckedIOException("Failed to close position delete source", e);
       }
   
       return indexes;
     }
   
     @Benchmark
     @Threads(1)
     public void stringEquals(Blackhole blackhole) {
       blackhole.consume(
           toPositionIndexesStringEquals(CloseableIterable.withNoopClose(deletes), null));
     }
   
     public static <T extends StructLike>
         CharSequenceMap<PositionDeleteIndex> toPositionIndexesStringEquals(
             CloseableIterable<T> posDeletes, DeleteFile file) {
       CharSequenceMap<PositionDeleteIndex> indexes = CharSequenceMap.create();
   
       try (CloseableIterable<T> deletes = posDeletes) {
         String lastFilePath = null;
         PositionDeleteIndex index = null;
         for (T delete : deletes) {
           CharSequence filePath = ((CharSequence) FILENAME_ACCESSOR.get(delete));
           long position = (long) POSITION_ACCESSOR.get(delete);
           if (lastFilePath == null || !lastFilePath.equals(filePath)) {
             lastFilePath = filePath.toString();
             index = indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));
           }
           index.delete(position);
         }
       } catch (IOException e) {
         throw new UncheckedIOException("Failed to close position delete source", e);
       }
   
       return indexes;
     }
   
     private record PositionDeleteRow(String path, long position) implements StructLike {
       @Override
       public int size() {
         return 2;
       }
   
       @Override
       @SuppressWarnings("unchecked")
       public <T> T get(int pos, Class<T> javaClass) {
         switch (pos) {
           case 0:
             return (T) path;
           case 1:
             return (T) (Long) position;
           default:
             throw new UnsupportedOperationException("Unsupported position: " + pos);
         }
       }
   
       @Override
       public <T> void set(int pos, T value) {
         throw new UnsupportedOperationException("Not supported");
       }
     }
   }
   ```
   </details>
   
   Related to #11648.
   Extracted from #15714.
   
   @anuragmantri


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "Neuw84 (via GitHub)" <gi...@apache.org>.
Neuw84 commented on PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#issuecomment-5831870247

   Some reference numbers with JMH (just discovered this after some JFR profiling), I had the patch ready but then I found this. It really pays the price.
   
   | Per delete row (3 forks × 10 iterations) | Stock | Reuse the last path's index |
   | --- | --- | --- |
   | Sorted by path, the spec's order | 213.0 ± 1.3 ns | **37.3 ± 0.3 ns** (5.7x) |
   | Interleaved, every row a new path | 272.9 ± 2.6 ns | 305.3 ± 14.7 ns (12% slower) |


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "huaxingao (via GitHub)" <gi...@apache.org>.
huaxingao merged PR #17864:
URL: https://github.com/apache/iceberg/pull/17864


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "anuragmantri (via GitHub)" <gi...@apache.org>.
anuragmantri commented on code in PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#discussion_r3926624178


##########
core/src/test/java/org/apache/iceberg/deletes/TestDeletes.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.deletes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.CharBuffer;
+import java.util.List;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.util.CharSequenceMap;
+import org.junit.jupiter.api.Test;
+
+public class TestDeletes {
+  static final String PATH_A = "file_a";
+  static final String PATH_B = "file_b";
+
+  @Test
+  public void testPositionIndexesWithNonStringPaths() {
+    // paths are typed as CharSequence, so readers are free to return any implementation
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(CharBuffer.wrap(PATH_A), 0L),
+            Row.of(CharBuffer.wrap(PATH_A), 5L),
+            Row.of(CharBuffer.wrap(PATH_B), 1L),
+            Row.of(CharBuffer.wrap(PATH_B), 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);
+    assertThat(collect(indexes.get(PATH_B))).containsExactlyInAnyOrder(1L, 2L);
+  }
+
+  @Test
+  public void testToPositionIndexesWithInterleavedPaths() {
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(PATH_A, 0L), Row.of(PATH_B, 1L), Row.of(PATH_A, 5L), Row.of(PATH_B, 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);

Review Comment:
   We should probably use the stricter `containsExactly()` because the bitmap is sorted and we want to make sure it stays that way.



##########
core/src/test/java/org/apache/iceberg/deletes/TestDeletes.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.deletes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.CharBuffer;
+import java.util.List;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.util.CharSequenceMap;
+import org.junit.jupiter.api.Test;
+
+public class TestDeletes {
+  static final String PATH_A = "file_a";
+  static final String PATH_B = "file_b";
+
+  @Test
+  public void testPositionIndexesWithNonStringPaths() {
+    // paths are typed as CharSequence, so readers are free to return any implementation
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(CharBuffer.wrap(PATH_A), 0L),
+            Row.of(CharBuffer.wrap(PATH_A), 5L),
+            Row.of(CharBuffer.wrap(PATH_B), 1L),
+            Row.of(CharBuffer.wrap(PATH_B), 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);
+    assertThat(collect(indexes.get(PATH_B))).containsExactlyInAnyOrder(1L, 2L);
+  }
+
+  public void testToPositionIndexesWithInterleavedPaths() {

Review Comment:
   Nit: we don't use `test*` prefixes anymore, can we call the tests `positionIndexesWithNonStringPaths` and `positionIndexesWithInterleavedPaths`?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "kinolaev (via GitHub)" <gi...@apache.org>.
kinolaev commented on PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#issuecomment-5502325136

   Thanks @anuragmantri! I've applied both suggestions and added you as a co-author.


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "anuragmantri (via GitHub)" <gi...@apache.org>.
anuragmantri commented on code in PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#discussion_r3906809905


##########
core/src/main/java/org/apache/iceberg/deletes/Deletes.java:
##########
@@ -147,11 +147,15 @@ public static <T extends StructLike> CharSequenceMap<PositionDeleteIndex> toPosi
     CharSequenceMap<PositionDeleteIndex> indexes = CharSequenceMap.create();
 
     try (CloseableIterable<T> deletes = posDeletes) {
+      String lastFilePath = null;
+      PositionDeleteIndex index = null;
       for (T delete : deletes) {
         CharSequence filePath = (CharSequence) FILENAME_ACCESSOR.get(delete);
         long position = (long) POSITION_ACCESSOR.get(delete);
-        PositionDeleteIndex index =
-            indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));
+        if (lastFilePath == null || !lastFilePath.equals(filePath)) {

Review Comment:
   [String.equals(Object)](https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/lang/String.html#equals(java.lang.Object)) returns `false` if the Object is not a String. In this case, the filePath is a `CharSequence`. So this comparison returns false. I see that all the format models return a String today but we may hit this in the future models. 
    
   ```suggestion
           if (lastFilePath == null || !lastFilePath.contentEquals(filePath)) {
   ```



##########
core/src/main/java/org/apache/iceberg/deletes/Deletes.java:
##########
@@ -147,11 +147,15 @@ public static <T extends StructLike> CharSequenceMap<PositionDeleteIndex> toPosi
     CharSequenceMap<PositionDeleteIndex> indexes = CharSequenceMap.create();
 
     try (CloseableIterable<T> deletes = posDeletes) {
+      String lastFilePath = null;
+      PositionDeleteIndex index = null;
       for (T delete : deletes) {
         CharSequence filePath = (CharSequence) FILENAME_ACCESSOR.get(delete);
         long position = (long) POSITION_ACCESSOR.get(delete);
-        PositionDeleteIndex index =
-            indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));
+        if (lastFilePath == null || !lastFilePath.equals(filePath)) {
+          lastFilePath = filePath.toString();
+          index = indexes.computeIfAbsent(filePath, key -> new BitmapPositionDeleteIndex(file));

Review Comment:
   It's worth adding a test for `toPositionIndexes`. Maybe something similar to:
   
   ```java
   @Test
       public void testPositionIndexesWithNonStringPaths() {
         // paths are typed as CharSequence, so readers are free to return Utf8 or any other
         // implementation; PATH_A is revisited after PATH_B to cover unsorted delete files
         List<Pair<CharSequence, Long>> rows =
             Lists.newArrayList(
                 Pair.of(CharBuffer.wrap(PATH_A), 0L),
                 Pair.of(CharBuffer.wrap(PATH_A), 5L),
                 Pair.of(CharBuffer.wrap(PATH_B), 1L),
                 Pair.of(CharBuffer.wrap(PATH_B), 2L),
                 Pair.of(CharBuffer.wrap(PATH_A), 9L));
   
         CharSequenceMap<PositionDeleteIndex> indexes =
             Deletes.toPositionIndexes(CloseableIterable.withNoopClose(toDeleteRows(rows)));
   
         assertThat(indexes).hasSize(2);
         assertThat(collect(indexes.get(PATH_A))).containsExactly(0L, 5L, 9L);
         assertThat(collect(indexes.get(PATH_B))).containsExactly(1L, 2L);
       }
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "huaxingao (via GitHub)" <gi...@apache.org>.
huaxingao commented on PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#issuecomment-5548081841

   THanks @kinolaev for the PR! Thanks @RussellSpitzer @anuragmantri for the review!


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#issuecomment-5528414848

   @anuragmantri Any other comments on this one?


-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "kinolaev (via GitHub)" <gi...@apache.org>.
kinolaev commented on code in PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#discussion_r3926676945


##########
core/src/test/java/org/apache/iceberg/deletes/TestDeletes.java:
##########
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.deletes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.CharBuffer;
+import java.util.List;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.util.CharSequenceMap;
+import org.junit.jupiter.api.Test;
+
+public class TestDeletes {
+  static final String PATH_A = "file_a";
+  static final String PATH_B = "file_b";
+
+  @Test
+  public void testPositionIndexesWithNonStringPaths() {
+    // paths are typed as CharSequence, so readers are free to return any implementation
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(CharBuffer.wrap(PATH_A), 0L),
+            Row.of(CharBuffer.wrap(PATH_A), 5L),
+            Row.of(CharBuffer.wrap(PATH_B), 1L),
+            Row.of(CharBuffer.wrap(PATH_B), 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);
+    assertThat(collect(indexes.get(PATH_B))).containsExactlyInAnyOrder(1L, 2L);
+  }
+
+  @Test
+  public void testToPositionIndexesWithInterleavedPaths() {
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(PATH_A, 0L), Row.of(PATH_B, 1L), Row.of(PATH_A, 5L), Row.of(PATH_B, 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);

Review Comment:
   Thanks @anuragmantri, both suggestions have been applied in 8ee08ba58c0955bb1b65a0d14c33f8db81552792



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "kinolaev (via GitHub)" <gi...@apache.org>.
kinolaev commented on code in PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#discussion_r3925919523


##########
core/src/test/java/org/apache/iceberg/deletes/TestDeletes.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.deletes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.CharBuffer;
+import java.util.List;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.util.CharSequenceMap;
+import org.junit.jupiter.api.Test;
+
+public class TestDeletes {
+  static final String PATH_A = "file_a";
+  static final String PATH_B = "file_b";
+
+  @Test
+  public void testPositionIndexesWithNonStringPaths() {
+    // paths are typed as CharSequence, so readers are free to return any implementation
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(CharBuffer.wrap(PATH_A), 0L),
+            Row.of(CharBuffer.wrap(PATH_A), 5L),
+            Row.of(CharBuffer.wrap(PATH_B), 1L),
+            Row.of(CharBuffer.wrap(PATH_B), 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);
+    assertThat(collect(indexes.get(PATH_B))).containsExactlyInAnyOrder(1L, 2L);
+  }
+
+  public void testToPositionIndexesWithInterleavedPaths() {

Review Comment:
   Not only that, but the test itself had an invalid assertion - what a shame! :) I'm sorry, and thanks for catching this, @RussellSpitzer. Fixed in 826511da1f87b3b10f0b531efc43e9d95eed7a46.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org


Re: [PR] Core: Avoid per-row path rehashing in toPositionIndexes [iceberg]

Posted by "RussellSpitzer (via GitHub)" <gi...@apache.org>.
RussellSpitzer commented on code in PR #17864:
URL: https://github.com/apache/iceberg/pull/17864#discussion_r3925686884


##########
core/src/test/java/org/apache/iceberg/deletes/TestDeletes.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.deletes;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.nio.CharBuffer;
+import java.util.List;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.TestHelpers.Row;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.util.CharSequenceMap;
+import org.junit.jupiter.api.Test;
+
+public class TestDeletes {
+  static final String PATH_A = "file_a";
+  static final String PATH_B = "file_b";
+
+  @Test
+  public void testPositionIndexesWithNonStringPaths() {
+    // paths are typed as CharSequence, so readers are free to return any implementation
+    List<StructLike> positionDeletes =
+        Lists.newArrayList(
+            Row.of(CharBuffer.wrap(PATH_A), 0L),
+            Row.of(CharBuffer.wrap(PATH_A), 5L),
+            Row.of(CharBuffer.wrap(PATH_B), 1L),
+            Row.of(CharBuffer.wrap(PATH_B), 2L));
+
+    CharSequenceMap<PositionDeleteIndex> indexes =
+        Deletes.toPositionIndexes(CloseableIterable.withNoopClose(positionDeletes), null);
+
+    assertThat(indexes).hasSize(2);
+    assertThat(collect(indexes.get(PATH_A))).containsExactlyInAnyOrder(0L, 5L);
+    assertThat(collect(indexes.get(PATH_B))).containsExactlyInAnyOrder(1L, 2L);
+  }
+
+  public void testToPositionIndexesWithInterleavedPaths() {

Review Comment:
   missing @Test annotation?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org


---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org