You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/12/16 21:19:15 UTC
[iceberg] branch master updated: Core: Add partition spec update
operation (#1942)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new a0e76df Core: Add partition spec update operation (#1942)
a0e76df is described below
commit a0e76df474e1d0d2ae9e7bb2f808f3dc3940a3d8
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Wed Dec 16 13:19:06 2020 -0800
Core: Add partition spec update operation (#1942)
Co-authored-by: Jun He <ju...@users.noreply.github.com>
---
api/src/main/java/org/apache/iceberg/Table.java | 7 ++
.../main/java/org/apache/iceberg/Transaction.java | 7 ++
.../org/apache/iceberg/UpdatePartitionSpec.java | 78 ++++++++++++++++++++++
.../java/org/apache/iceberg/BaseMetadataTable.java | 5 ++
.../main/java/org/apache/iceberg/BaseTable.java | 4 ++
.../java/org/apache/iceberg/BaseTransaction.java | 13 ++++
.../apache/iceberg/BaseUpdatePartitionSpec.java | 12 +++-
.../apache/iceberg/CommitCallbackTransaction.java | 5 ++
.../java/org/apache/iceberg/TableMetadata.java | 6 +-
.../java/org/apache/iceberg/TestTableMetadata.java | 2 +-
10 files changed, 133 insertions(+), 6 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/Table.java b/api/src/main/java/org/apache/iceberg/Table.java
index ec09880..a4557c8 100644
--- a/api/src/main/java/org/apache/iceberg/Table.java
+++ b/api/src/main/java/org/apache/iceberg/Table.java
@@ -139,6 +139,13 @@ public interface Table {
UpdateSchema updateSchema();
/**
+ * Create a new {@link UpdatePartitionSpec} to alter the partition spec of this table and commit the change.
+ *
+ * @return a new {@link UpdatePartitionSpec}
+ */
+ UpdatePartitionSpec updateSpec();
+
+ /**
* Create a new {@link UpdateProperties} to update table properties and commit the changes.
*
* @return a new {@link UpdateProperties}
diff --git a/api/src/main/java/org/apache/iceberg/Transaction.java b/api/src/main/java/org/apache/iceberg/Transaction.java
index 63401cf..609f86d 100644
--- a/api/src/main/java/org/apache/iceberg/Transaction.java
+++ b/api/src/main/java/org/apache/iceberg/Transaction.java
@@ -41,6 +41,13 @@ public interface Transaction {
UpdateSchema updateSchema();
/**
+ * Create a new {@link UpdatePartitionSpec} to alter the partition spec of this table.
+ *
+ * @return a new {@link UpdatePartitionSpec}
+ */
+ UpdatePartitionSpec updateSpec();
+
+ /**
* Create a new {@link UpdateProperties} to update table properties.
*
* @return a new {@link UpdateProperties}
diff --git a/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java b/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
index d15c272..5e85dd9 100644
--- a/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/UpdatePartitionSpec.java
@@ -20,6 +20,7 @@
package org.apache.iceberg;
import org.apache.iceberg.exceptions.CommitFailedException;
+import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Term;
/**
@@ -29,15 +30,92 @@ import org.apache.iceberg.expressions.Term;
* will not be resolved and will result in a {@link CommitFailedException}.
*/
public interface UpdatePartitionSpec extends PendingUpdate<PartitionSpec> {
+ /**
+ * Set whether column resolution in the source schema should be case sensitive.
+ *
+ * @param isCaseSensitive whether column resolution should be case sensitive
+ * @return this for method chaining
+ */
+ UpdatePartitionSpec caseSensitive(boolean isCaseSensitive);
+
+ /**
+ * Add a new partition field from a source column.
+ * <p>
+ * The partition field will be created as an identity partition field for the given source column, with the same name
+ * as the source column.
+ * <p>
+ * The source column is located using {@link Schema#findField(String)}.
+ *
+ * @param sourceName source column name in the table schema
+ * @return this for method chaining
+ * @throws IllegalArgumentException If the an identity partition field for the source already exists, or if this
+ * change conflicts with other additions, removals, or renames.
+ */
UpdatePartitionSpec addField(String sourceName);
+ /**
+ * Add a new partition field from an {@link Expressions expression term}.
+ * <p>
+ * The partition field will use the term's transform or the identity transform if the term is a reference.
+ * <p>
+ * The term's reference is used to locate the source column using {@link Schema#findField(String)}.
+ * <p>
+ * The new partition field will be named for the source column and the transform.
+ *
+ * @param term source column name in the table schema
+ * @return this for method chaining
+ * @throws IllegalArgumentException If the a partition field for the transform and source already exists, or if this
+ * change conflicts with other additions, removals, or renames.
+ */
UpdatePartitionSpec addField(Term term);
+ /**
+ * Add a new partition field from an {@link Expressions expression term}, with the given partition field name.
+ * <p>
+ * The partition field will use the term's transform or the identity transform if the term is a reference.
+ * <p>
+ * The term's reference is used to locate the source column using {@link Schema#findField(String)}.
+ *
+ * @param name name for the partition field
+ * @param term expression for the partition transform
+ * @return this for method chaining
+ * @throws IllegalArgumentException If the a partition field for the transform and source already exists, if a
+ * partition field with the given name already exists, or if this change conflicts
+ * with other additions, removals, or renames.
+ */
UpdatePartitionSpec addField(String name, Term term);
+ /**
+ * Remove a partition field by name.
+ *
+ * @param name name of the partition field to remove
+ * @return this for method chaining
+ * @throws IllegalArgumentException If the a partition field with the given name does not exist, or if this change
+ * conflicts with other additions, removals, or renames.
+ */
UpdatePartitionSpec removeField(String name);
+ /**
+ * Remove a partition field by its transform {@link Expressions expression term}.
+ * <p>
+ * The partition field with the same transform and source reference will be removed. If the term is a reference and
+ * does not have a transform, the identity transform is used.
+ *
+ * @param term expression for the partition transform to remove
+ * @return this for method chaining
+ * @throws IllegalArgumentException If the a partition field with the given transform and source does not exist, or
+ * if this change conflicts with other additions, removals, or renames.
+ */
UpdatePartitionSpec removeField(Term term);
+ /**
+ * Rename a field in the partition spec.
+ *
+ * @param name name of the partition field to rename
+ * @param newName replacement name for the partition field
+ * @return this for method chaining
+ * @throws IllegalArgumentException If name doesn't identify a column in the schema or if this change conflicts with
+ * other additions, removals, or renames.
+ */
UpdatePartitionSpec renameField(String name, String newName);
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
index 40f8ea7..ec08120 100644
--- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
+++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
@@ -108,6 +108,11 @@ abstract class BaseMetadataTable implements Table {
}
@Override
+ public UpdatePartitionSpec updateSpec() {
+ throw new UnsupportedOperationException("Cannot update the partition spec of a metadata table");
+ }
+
+ @Override
public UpdateProperties updateProperties() {
throw new UnsupportedOperationException("Cannot update the properties of a metadata table");
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseTable.java b/core/src/main/java/org/apache/iceberg/BaseTable.java
index 5f4e4e7..005fba0 100644
--- a/core/src/main/java/org/apache/iceberg/BaseTable.java
+++ b/core/src/main/java/org/apache/iceberg/BaseTable.java
@@ -119,6 +119,10 @@ public class BaseTable implements Table, HasTableOperations {
return new SchemaUpdate(ops);
}
+ public UpdatePartitionSpec updateSpec() {
+ return new BaseUpdatePartitionSpec(ops);
+ }
+
@Override
public UpdateProperties updateProperties() {
return new PropertiesUpdate(ops);
diff --git a/core/src/main/java/org/apache/iceberg/BaseTransaction.java b/core/src/main/java/org/apache/iceberg/BaseTransaction.java
index 750962a..a004854 100644
--- a/core/src/main/java/org/apache/iceberg/BaseTransaction.java
+++ b/core/src/main/java/org/apache/iceberg/BaseTransaction.java
@@ -103,6 +103,14 @@ class BaseTransaction implements Transaction {
}
@Override
+ public UpdatePartitionSpec updateSpec() {
+ checkLastOperationCommitted("UpdateSpec");
+ UpdatePartitionSpec partitionSpecChange = new BaseUpdatePartitionSpec(transactionOps);
+ updates.add(partitionSpecChange);
+ return partitionSpecChange;
+ }
+
+ @Override
public UpdateProperties updateProperties() {
checkLastOperationCommitted("UpdateProperties");
UpdateProperties props = new PropertiesUpdate(transactionOps);
@@ -568,6 +576,11 @@ class BaseTransaction implements Transaction {
}
@Override
+ public UpdatePartitionSpec updateSpec() {
+ return BaseTransaction.this.updateSpec();
+ }
+
+ @Override
public UpdateProperties updateProperties() {
return BaseTransaction.this.updateProperties();
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
index efcf089..0818fb3 100644
--- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
+++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
@@ -43,7 +43,6 @@ import org.apache.iceberg.util.Pair;
class BaseUpdatePartitionSpec implements UpdatePartitionSpec {
private final TableOperations ops;
- private final boolean caseSensitive;
private final TableMetadata base;
private final int formatVersion;
private final PartitionSpec spec;
@@ -58,11 +57,12 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec {
private final Set<Object> deletes = Sets.newHashSet();
private final Map<String, String> renames = Maps.newHashMap();
+ private boolean caseSensitive;
private int lastAssignedPartitionId;
- BaseUpdatePartitionSpec(TableOperations ops, boolean caseSensitive) {
+ BaseUpdatePartitionSpec(TableOperations ops) {
this.ops = ops;
- this.caseSensitive = caseSensitive;
+ this.caseSensitive = true;
this.base = ops.current();
this.formatVersion = base.formatVersion();
this.spec = base.spec();
@@ -109,6 +109,12 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec {
}
@Override
+ public UpdatePartitionSpec caseSensitive(boolean isCaseSensitive) {
+ this.caseSensitive = isCaseSensitive;
+ return this;
+ }
+
+ @Override
public BaseUpdatePartitionSpec addField(String sourceName) {
return addField(Expressions.ref(sourceName));
}
diff --git a/core/src/main/java/org/apache/iceberg/CommitCallbackTransaction.java b/core/src/main/java/org/apache/iceberg/CommitCallbackTransaction.java
index 0abb5c1..bd15744 100644
--- a/core/src/main/java/org/apache/iceberg/CommitCallbackTransaction.java
+++ b/core/src/main/java/org/apache/iceberg/CommitCallbackTransaction.java
@@ -43,6 +43,11 @@ class CommitCallbackTransaction implements Transaction {
}
@Override
+ public UpdatePartitionSpec updateSpec() {
+ return wrapped.updateSpec();
+ }
+
+ @Override
public UpdateProperties updateProperties() {
return wrapped.updateProperties();
}
diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java
index bbfb10a..7aa23c8 100644
--- a/core/src/main/java/org/apache/iceberg/TableMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java
@@ -475,8 +475,10 @@ public class TableMetadata implements Serializable {
}
}
- Preconditions.checkArgument(defaultSpecId != newDefaultSpecId,
- "Cannot set default partition spec to the current default");
+ if (defaultSpecId == newDefaultSpecId) {
+ // the new spec is already current and no change is needed
+ return this;
+ }
ImmutableList.Builder<PartitionSpec> builder = ImmutableList.<PartitionSpec>builder()
.addAll(specs);
diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
index dfbbbf0..7509ac4 100644
--- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
@@ -501,7 +501,7 @@ public class TestTableMetadata {
PartitionSpec spec = PartitionSpec.builderFor(schema).withSpecId(5)
.add(3, 1005, "x_partition", "bucket[4]")
- .add(5, 1005, "z_partition", "bucket[8]")
+ .add(5, 1003, "z_partition", "bucket[8]")
.build();
String location = "file://tmp/db/table";
TableMetadata metadata = TableMetadata.newTableMetadata(schema, spec, location, ImmutableMap.of());