You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2021/05/13 23:20:18 UTC
[GitHub] [iceberg] aokolnychyi commented on a change in pull request #2591: Spark: RewriteDatafilesAction V2
aokolnychyi commented on a change in pull request #2591:
URL: https://github.com/apache/iceberg/pull/2591#discussion_r632116703
##########
File path: api/src/main/java/org/apache/iceberg/actions/RewriteStrategy.java
##########
@@ -27,7 +27,7 @@
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.Table;
-interface RewriteStrategy extends Serializable {
+public interface RewriteStrategy extends Serializable {
Review comment:
Shall we move it to core for now? I think it is a bit too early to expose it.
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
Review comment:
Would it make sense to put this into a separate method to simplify the collector statement?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
Review comment:
Typo? `committerService`?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
Review comment:
Or we actually assume this only happens if all rewrites have completed?
##########
File path: api/src/main/java/org/apache/iceberg/actions/RewriteStrategy.java
##########
@@ -71,8 +71,9 @@
* Method which will rewrite files based on this particular RewriteStrategy's algorithm.
* This will most likely be Action framework specific (Spark/Presto/Flink ....).
*
+ * @param setId an identifier for this set of files
* @param filesToRewrite a group of files to be rewritten together
* @return a list of newly written files
*/
- List<DataFile> rewriteFiles(List<FileScanTask> filesToRewrite);
+ Set<DataFile> rewriteFiles(String setId, List<FileScanTask> filesToRewrite);
Review comment:
nit: `setID`
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
Review comment:
nit: extra space, do we need an explicit `this` in `this.options()`?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
Review comment:
Seems like we don't close this `Iterable` anywhere?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
Review comment:
Can we split this method into smaller ones? I think planning file groups could definitely be in another method.
##########
File path: api/src/main/java/org/apache/iceberg/io/CloseableIterable.java
##########
@@ -26,9 +26,11 @@
import java.util.NoSuchElementException;
import java.util.function.Function;
import java.util.function.Predicate;
+import java.util.stream.Stream;
Review comment:
Not needed?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
Review comment:
I'd consider putting this into a separate method and calling it from map.
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
Review comment:
nit: I think it is a bit more readable to have `.stream()` on the same line with `=` and then each subsequent action on a separate line.
```
... filesByPartition = Streams.stream(files)
.collect(...);
```
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
Review comment:
nit: `Maps.newHashMap`?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
Review comment:
Is this used anywhere?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
Review comment:
Here as well. It may be a bit easier to read if Collectors.toMap was on a single separate line.
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
Review comment:
Is there a better prefix we can give? `my` -> `current` or something?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
Review comment:
nit: shall we define temp vars for UUID.randomUUID() and FileGroupInfo?
```
String xxx = UUID.randomUUID().toString();
FileGroupInfo yyy = new FileGroupInfo(xxx, myJobIndex, myPartIndex, e.getKey());
return Pair.of(yyy, tasks);
```
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
Review comment:
`completedRewrite` -> `completedRewrites`?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
Review comment:
Can we move `totalGroups` closer to the place it is used?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
Review comment:
Should this happen only if partial progress enabled?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
Review comment:
nit: make final and move above mutable vars?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
Review comment:
Shall we stop other rewrites if this happens?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
Review comment:
I am not sure I understand the second branch of this condition. Suppose I have 10 partitions and allow only 2 commits at most. If I managed to compact 9 partitions and one failed, we should still commit the remaining 4?
##########
File path: spark3/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSpark3.java
##########
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.Set;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.spark.FileRewriteCoordinator;
+import org.apache.iceberg.spark.actions.rewrite.Spark3BinPackStrategy;
+import org.apache.spark.sql.SparkSession;
+
+public class RewriteDataFilesSpark3 extends BaseRewriteDataFilesSparkAction {
+ private FileRewriteCoordinator coordinator = FileRewriteCoordinator.get();
Review comment:
nit: final?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Cannot commit rewrite and partial progress not enabled, removing all completed work", e);
+ completedCommits.forEach(this::abortFileGroup);
+ }
+ }
+ });
+
+ // Start rewrite tasks
+ Tasks.Builder<Pair<FileGroupInfo, List<FileScanTask>>> rewriteTaskBuilder = Tasks.foreach(jobStream.iterator())
+ .suppressFailureWhenFinished()
+ .executeWith(rewriteService)
+ .noRetry()
+ .onFailure((info, exception) -> {
+ LOG.error("Failure during rewrite process for group {}", info.first(), exception);
+ abortFileGroup(info.first().setId);
+ });
+
+ if (partialProgressEnabled) {
+ rewriteTaskBuilder = rewriteTaskBuilder.suppressFailureWhenFinished();
+ } else {
+ rewriteTaskBuilder = rewriteTaskBuilder.stopOnFailure();
+ }
+
+ rewriteTaskBuilder
+ .run(i -> {
+ String setId = i.first().setId();
+ Set<DataFile> addedFiles = strategy.rewriteFiles(setId, i.second());
+ completedRewrite.offer(setId);
+ results.put(setId, Pair.of(i.first(), new FileGroupRewriteResult(addedFiles.size(), i.second().size())));
+ });
+ stillRewriting.set(false);
+ commiterService.shutdown();
+
+ try {
+ commiterService.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new Result(results.values().stream().collect(Collectors.toMap(p -> p.first(), p -> p.second())));
+ }
+
+ protected Table table() {
+ return table;
+ }
+
+ /**
+ * Perform a commit operation on the table adding and removing files as required for this set of file groups,
+ * on failure should clean up and rethrow exception
+ * @param completedSetIds fileSets to commit
+ */
+ protected abstract void commitFileGroups(Set<String> completedSetIds);
+
+ /**
+ * Clean up a specified file set by removing any files created for that operation
+ * @param setId fileSet to clean
+ */
+ protected abstract void abortFileGroup(String setId);
+
+ class Result implements RewriteDataFiles.Result {
Review comment:
Would it make sense to move this to core where we have other results?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
Review comment:
Can we put the commit closure into a separate method?
##########
File path: spark3/src/main/java/org/apache/iceberg/spark/actions/RewriteDataFilesSpark3.java
##########
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.Set;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.spark.FileRewriteCoordinator;
+import org.apache.iceberg.spark.actions.rewrite.Spark3BinPackStrategy;
+import org.apache.spark.sql.SparkSession;
+
+public class RewriteDataFilesSpark3 extends BaseRewriteDataFilesSparkAction {
Review comment:
`BaseRewriteDataFilesSpark3Action`?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
Review comment:
Can we group all abstract method together? I think they should also be below the constructor.
##########
File path: spark3/src/main/java/org/apache/iceberg/spark/actions/rewrite/Spark3BinPackStrategy.java
##########
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions.rewrite;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.BinPackStrategy;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.spark.FileRewriteCoordinator;
+import org.apache.iceberg.spark.FileScanTaskSetManager;
+import org.apache.iceberg.spark.SparkReadOptions;
+import org.apache.iceberg.spark.SparkWriteOptions;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+
+public class Spark3BinPackStrategy extends BinPackStrategy {
+ private final Table table;
+ private final SparkSession spark;
+ private final FileScanTaskSetManager manager = FileScanTaskSetManager.get();
+
+ public Spark3BinPackStrategy(Table table, SparkSession spark) {
+ this.table = table;
+ this.spark = spark;
+ }
+
+
+ @Override
+ public Table table() {
+ return table;
+ }
+
+ @Override
+ public Set<DataFile> rewriteFiles(String setId, List<FileScanTask> filesToRewrite) {
+ manager.stageTasks(table, setId, filesToRewrite);
+
+ // read and pack original 4 files into 2 splits
+ Dataset<Row> scanDF = spark.read().format("iceberg")
+ .option(SparkReadOptions.FILE_SCAN_TASK_SET_ID, setId)
+ .option(SparkReadOptions.SPLIT_SIZE, Long.toString(targetFileSize()))
+ .option(SparkReadOptions.FILE_OPEN_COST, "0")
+ .load(table.name());
+
+ // write the packed data into new files where each split becomes a new file
+ try {
+ scanDF.write()
Review comment:
I think we should clone the original session and disable AQE to avoid any surprises.
##########
File path: spark3/src/main/java/org/apache/iceberg/spark/actions/rewrite/Spark3BinPackStrategy.java
##########
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions.rewrite;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.BinPackStrategy;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.spark.FileRewriteCoordinator;
+import org.apache.iceberg.spark.FileScanTaskSetManager;
+import org.apache.iceberg.spark.SparkReadOptions;
+import org.apache.iceberg.spark.SparkWriteOptions;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+
+public class Spark3BinPackStrategy extends BinPackStrategy {
+ private final Table table;
+ private final SparkSession spark;
+ private final FileScanTaskSetManager manager = FileScanTaskSetManager.get();
+
+ public Spark3BinPackStrategy(Table table, SparkSession spark) {
+ this.table = table;
+ this.spark = spark;
+ }
+
+
+ @Override
+ public Table table() {
+ return table;
+ }
+
+ @Override
+ public Set<DataFile> rewriteFiles(String setId, List<FileScanTask> filesToRewrite) {
Review comment:
I think we should annotate each job with proper Spark UI.
```
Rewriting files (bin-pack, partition '$partition', jobIndex/jobs, partitionIndex/partitions) in db.tbl
```
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Cannot commit rewrite and partial progress not enabled, removing all completed work", e);
+ completedCommits.forEach(this::abortFileGroup);
+ }
+ }
+ });
+
+ // Start rewrite tasks
+ Tasks.Builder<Pair<FileGroupInfo, List<FileScanTask>>> rewriteTaskBuilder = Tasks.foreach(jobStream.iterator())
+ .suppressFailureWhenFinished()
+ .executeWith(rewriteService)
+ .noRetry()
+ .onFailure((info, exception) -> {
+ LOG.error("Failure during rewrite process for group {}", info.first(), exception);
+ abortFileGroup(info.first().setId);
+ });
+
+ if (partialProgressEnabled) {
+ rewriteTaskBuilder = rewriteTaskBuilder.suppressFailureWhenFinished();
+ } else {
+ rewriteTaskBuilder = rewriteTaskBuilder.stopOnFailure();
+ }
+
+ rewriteTaskBuilder
+ .run(i -> {
+ String setId = i.first().setId();
+ Set<DataFile> addedFiles = strategy.rewriteFiles(setId, i.second());
+ completedRewrite.offer(setId);
+ results.put(setId, Pair.of(i.first(), new FileGroupRewriteResult(addedFiles.size(), i.second().size())));
+ });
+ stillRewriting.set(false);
+ commiterService.shutdown();
+
+ try {
+ commiterService.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new Result(results.values().stream().collect(Collectors.toMap(p -> p.first(), p -> p.second())));
Review comment:
nit: `Collectors.toMap(Pair::first, Pair::second)`
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
Review comment:
Shouldn't it be `>=`, though? We should be able to commit if we have enough groups, no?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Cannot commit rewrite and partial progress not enabled, removing all completed work", e);
+ completedCommits.forEach(this::abortFileGroup);
+ }
+ }
+ });
+
+ // Start rewrite tasks
+ Tasks.Builder<Pair<FileGroupInfo, List<FileScanTask>>> rewriteTaskBuilder = Tasks.foreach(jobStream.iterator())
+ .suppressFailureWhenFinished()
+ .executeWith(rewriteService)
+ .noRetry()
+ .onFailure((info, exception) -> {
+ LOG.error("Failure during rewrite process for group {}", info.first(), exception);
+ abortFileGroup(info.first().setId);
+ });
+
+ if (partialProgressEnabled) {
+ rewriteTaskBuilder = rewriteTaskBuilder.suppressFailureWhenFinished();
+ } else {
+ rewriteTaskBuilder = rewriteTaskBuilder.stopOnFailure();
+ }
+
+ rewriteTaskBuilder
+ .run(i -> {
+ String setId = i.first().setId();
+ Set<DataFile> addedFiles = strategy.rewriteFiles(setId, i.second());
+ completedRewrite.offer(setId);
+ results.put(setId, Pair.of(i.first(), new FileGroupRewriteResult(addedFiles.size(), i.second().size())));
+ });
+ stillRewriting.set(false);
+ commiterService.shutdown();
+
+ try {
+ commiterService.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
Review comment:
A helper error message?
##########
File path: spark3/src/main/java/org/apache/iceberg/spark/actions/rewrite/Spark3BinPackStrategy.java
##########
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions.rewrite;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.BinPackStrategy;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
+import org.apache.iceberg.spark.FileRewriteCoordinator;
+import org.apache.iceberg.spark.FileScanTaskSetManager;
+import org.apache.iceberg.spark.SparkReadOptions;
+import org.apache.iceberg.spark.SparkWriteOptions;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.catalyst.analysis.NoSuchTableException;
+
+public class Spark3BinPackStrategy extends BinPackStrategy {
+ private final Table table;
+ private final SparkSession spark;
+ private final FileScanTaskSetManager manager = FileScanTaskSetManager.get();
+
+ public Spark3BinPackStrategy(Table table, SparkSession spark) {
+ this.table = table;
+ this.spark = spark;
+ }
+
+
+ @Override
+ public Table table() {
+ return table;
+ }
+
+ @Override
+ public Set<DataFile> rewriteFiles(String setId, List<FileScanTask> filesToRewrite) {
+ manager.stageTasks(table, setId, filesToRewrite);
+
+ // read and pack original 4 files into 2 splits
Review comment:
nit: wrong comment?
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
+ Thread.yield();
+ if (completedRewrite.size() > groupsPerCommit) {
+ // Gather a group of rewrites to do a commit
+ Set<String> batch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ for (int i = 0; i < groupsPerCommit; i++) {
+ batch.add(completedRewrite.poll());
+ }
+ // Running commit on the group we gathered
+ try {
+ commitFileGroups(batch);
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Failure during rewrite commit process, partial progress not enabled. Rethrowing", e);
+ throw e;
+ } else {
+ LOG.error("Failure during rewrite commit process, partial progress enabled. Ignoring", e);
+ }
+ }
+ completedCommits.addAll(batch);
+ }
+ }
+ Set<String> finalBatch = Sets.newHashSetWithExpectedSize(groupsPerCommit);
+ while (completedRewrite.size() != 0) {
+ finalBatch.add(completedRewrite.poll());
+ }
+ if (finalBatch.size() != 0) {
+ // Final Commit
+ commitFileGroups(finalBatch);
+ completedCommits.addAll(finalBatch);
+ }
+ } catch (Exception e) {
+ if (!partialProgressEnabled) {
+ LOG.error("Cannot commit rewrite and partial progress not enabled, removing all completed work", e);
+ completedCommits.forEach(this::abortFileGroup);
+ }
+ }
+ });
+
+ // Start rewrite tasks
+ Tasks.Builder<Pair<FileGroupInfo, List<FileScanTask>>> rewriteTaskBuilder = Tasks.foreach(jobStream.iterator())
+ .suppressFailureWhenFinished()
+ .executeWith(rewriteService)
+ .noRetry()
+ .onFailure((info, exception) -> {
+ LOG.error("Failure during rewrite process for group {}", info.first(), exception);
+ abortFileGroup(info.first().setId);
+ });
+
+ if (partialProgressEnabled) {
+ rewriteTaskBuilder = rewriteTaskBuilder.suppressFailureWhenFinished();
+ } else {
+ rewriteTaskBuilder = rewriteTaskBuilder.stopOnFailure();
+ }
+
+ rewriteTaskBuilder
+ .run(i -> {
+ String setId = i.first().setId();
+ Set<DataFile> addedFiles = strategy.rewriteFiles(setId, i.second());
+ completedRewrite.offer(setId);
+ results.put(setId, Pair.of(i.first(), new FileGroupRewriteResult(addedFiles.size(), i.second().size())));
+ });
+ stillRewriting.set(false);
+ commiterService.shutdown();
+
+ try {
+ commiterService.awaitTermination(10, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return new Result(results.values().stream().collect(Collectors.toMap(p -> p.first(), p -> p.second())));
+ }
+
+ protected Table table() {
+ return table;
+ }
+
+ /**
+ * Perform a commit operation on the table adding and removing files as required for this set of file groups,
+ * on failure should clean up and rethrow exception
+ * @param completedSetIds fileSets to commit
+ */
+ protected abstract void commitFileGroups(Set<String> completedSetIds);
+
+ /**
+ * Clean up a specified file set by removing any files created for that operation
+ * @param setId fileSet to clean
+ */
+ protected abstract void abortFileGroup(String setId);
Review comment:
Let's move this directly after the constructor together with other abstract methods.
##########
File path: spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java
##########
@@ -0,0 +1,321 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.spark.actions;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.actions.RewriteDataFiles;
+import org.apache.iceberg.actions.RewriteStrategy;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Expressions;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.relocated.com.google.common.collect.Streams;
+import org.apache.iceberg.relocated.com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.iceberg.util.Pair;
+import org.apache.iceberg.util.PropertyUtil;
+import org.apache.iceberg.util.Tasks;
+import org.apache.spark.sql.SparkSession;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+abstract class BaseRewriteDataFilesSparkAction
+ extends BaseSnapshotUpdateSparkAction<RewriteDataFiles, RewriteDataFiles.Result> implements RewriteDataFiles {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BaseRewriteDataFilesSparkAction.class);
+
+ private RewriteDataFiles.Strategy strategyType = Strategy.BINPACK;
+ private Table table;
+ private Expression filter = Expressions.alwaysTrue();
+
+ /**
+ * returns the Spark version specific strategy
+ */
+ protected abstract RewriteStrategy rewriteStrategy(Strategy type);
+
+ protected BaseRewriteDataFilesSparkAction(SparkSession spark, Table table) {
+ super(spark);
+ this.table = table;
+ }
+
+ @Override
+ public RewriteDataFiles strategy(Strategy type) {
+ strategyType = type;
+ return this;
+ }
+
+ @Override
+ public RewriteDataFiles filter(Expression expression) {
+ filter = Expressions.and(filter, expression);
+ return this;
+ }
+
+ @Override
+ public Result execute() {
+ RewriteStrategy strategy = rewriteStrategy(strategyType).options(this.options());
+
+ CloseableIterable<FileScanTask> files = table.newScan()
+ .filter(filter)
+ .ignoreResiduals()
+ .planFiles();
+
+ Map<StructLike, List<FileScanTask>> filesByPartition =
+ Streams.stream(files).collect(Collectors.groupingBy(task -> task.file().partition()));
+
+ Map<StructLike, List<List<FileScanTask>>> fileGroupsByPartition =
+ filesByPartition.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> {
+ Iterable<FileScanTask> filtered = strategy.selectFilesToRewrite(e.getValue());
+ Iterable<List<FileScanTask>> groupedTasks = strategy.planFileGroups(filtered);
+ return ImmutableList.copyOf(groupedTasks);
+ }));
+
+ Map<StructLike, Integer> numGroupsPerPartition = fileGroupsByPartition.entrySet().stream().collect(Collectors.toMap(
+ e -> e.getKey(),
+ e -> e.getValue().size()
+ ));
+
+ Integer totalGroups = numGroupsPerPartition.values().stream().reduce(Integer::sum).orElse(0);
+ Integer totalPartitions = numGroupsPerPartition.keySet().size();
+
+ Map<StructLike, Integer> partitionIndex = new HashMap<>();
+ AtomicInteger jobIndex = new AtomicInteger(1);
+
+ // Todo Check if we need to randomize the order in which we do jobs, instead of being partition centric
+ Stream<Pair<FileGroupInfo, List<FileScanTask>>> jobStream = fileGroupsByPartition.entrySet().stream().flatMap(
+ e -> e.getValue().stream().map(tasks -> {
+ int myJobIndex = jobIndex.getAndIncrement();
+ int myPartIndex = partitionIndex.merge(e.getKey(), 1, Integer::sum);
+ return Pair.of(new FileGroupInfo(UUID.randomUUID().toString(), myJobIndex, myPartIndex, e.getKey()), tasks);
+ })
+ );
+
+ int maxConcurrentFileGroupActions = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS,
+ RewriteDataFiles.MAX_CONCURRENT_FILE_GROUP_ACTIONS_DEFAULT);
+
+ int maxCommits = PropertyUtil.propertyAsInt(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS,
+ RewriteDataFiles.PARTIAL_PROGRESS_MAX_COMMITS_DEFAULT);
+
+ boolean partialProgressEnabled = PropertyUtil.propertyAsBoolean(options(),
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED,
+ RewriteDataFiles.PARTIAL_PROGRESS_ENABLED_DEFAULT);
+
+ int groupsPerCommit = partialProgressEnabled ? totalGroups / maxCommits : totalGroups;
+
+ AtomicBoolean stillRewriting = new AtomicBoolean(true);
+ ConcurrentLinkedQueue<String> completedRewrite = new ConcurrentLinkedQueue<>();
+ ConcurrentHashMap<String, Pair<FileGroupInfo, FileGroupRewriteResult>> results = new ConcurrentHashMap<>();
+ ConcurrentLinkedQueue<String> completedCommits = new ConcurrentLinkedQueue<>();
+
+ ExecutorService rewriteService = Executors.newFixedThreadPool(maxConcurrentFileGroupActions,
+ new ThreadFactoryBuilder().setNameFormat("Rewrite-Service-%d").build());
+
+ ExecutorService commiterService = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setNameFormat("Committer-Service").build());
+
+ commiterService.execute(() -> {
+ try {
+ while (stillRewriting.get() || completedRewrite.size() > groupsPerCommit) {
Review comment:
Wait, I think that's handled below.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org