[GitHub] [iceberg] openinx commented on a change in pull request #1843: Support for file paths in SparkCatalogs via HadoopTables

[GitBox <gi...@apache.org>]

openinx commented on a change in pull request #1843:
URL: https://github.com/apache/iceberg/pull/1843#discussion_r550101676



##########
File path: core/src/main/java/org/apache/iceberg/hadoop/HadoopTables.java
##########
@@ -200,6 +193,165 @@ TableOperations newTableOps(String location) {
     }
   }
 
+  private TableMetadata tableMetadata(Schema schema, PartitionSpec spec, SortOrder order,
+                                      Map<String, String> properties, String location) {
+    Preconditions.checkNotNull(schema, "A table schema is required");
+
+    Map<String, String> tableProps = properties == null ? ImmutableMap.of() : properties;
+    PartitionSpec partitionSpec = spec == null ? PartitionSpec.unpartitioned() : spec;
+    SortOrder sortOrder = order == null ? SortOrder.unsorted() : order;
+    return TableMetadata.newTableMetadata(schema, partitionSpec, sortOrder, location, tableProps);
+  }
+
+  /**
+   * Start a transaction to create a table.
+   *
+   * @param location a location for the table
+   * @param schema a schema
+   * @param spec a partition spec
+   * @param properties a string map of table properties
+   * @return a {@link Transaction} to create the table
+   * @throws AlreadyExistsException if the table already exists
+   */
+  public Transaction newCreateTableTransaction(
+      String location,
+      Schema schema,
+      PartitionSpec spec,
+      Map<String, String> properties) {
+    return buildTable(location, schema).withPartitionSpec(spec).withProperties(properties).createTransaction();
+  }
+
+  /**
+   * Start a transaction to replace a table.
+   *
+   * @param location a location for the table
+   * @param schema a schema
+   * @param spec a partition spec
+   * @param properties a string map of table properties
+   * @param orCreate whether to create the table if not exists
+   * @return a {@link Transaction} to replace the table
+   * @throws NoSuchTableException if the table doesn't exist and orCreate is false
+   */
+  public Transaction newReplaceTableTransaction(
+      String location,
+      Schema schema,
+      PartitionSpec spec,
+      Map<String, String> properties,
+      boolean orCreate) {
+
+
+    Catalog.TableBuilder builder = buildTable(location, schema).withPartitionSpec(spec).withProperties(properties);
+    return orCreate ? builder.createOrReplaceTransaction() : builder.replaceTransaction();
+  }
+
+  public Catalog.TableBuilder buildTable(String location, Schema schema) {
+    return new HadoopTableBuilder(location, schema);
+  }
+
+  private class HadoopTableBuilder implements Catalog.TableBuilder {
+    private final String location;
+    private final Schema schema;
+    private final ImmutableMap.Builder<String, String> propertiesBuilder = ImmutableMap.builder();
+    private PartitionSpec spec = PartitionSpec.unpartitioned();
+    private SortOrder sortOrder = SortOrder.unsorted();
+
+
+    HadoopTableBuilder(String location, Schema schema) {
+      this.location = location;
+      this.schema = schema;
+    }
+
+    @Override
+    public Catalog.TableBuilder withPartitionSpec(PartitionSpec newSpec) {
+      this.spec = newSpec != null ? newSpec : PartitionSpec.unpartitioned();
+      return this;
+    }
+
+    @Override
+    public Catalog.TableBuilder withSortOrder(SortOrder newSortOrder) {
+      this.sortOrder = newSortOrder != null ? newSortOrder : SortOrder.unsorted();
+      return this;
+    }
+
+    @Override
+    public Catalog.TableBuilder withLocation(String newLocation) {
+      Preconditions.checkArgument(newLocation == null || location.equals(newLocation),
+          String.format("Table location %s differs from the table location (%s) from the PathIdentifier",
+              newLocation, location));
+      return this;
+    }
+
+    @Override
+    public Catalog.TableBuilder withProperties(Map<String, String> properties) {
+      if (properties != null) {
+        propertiesBuilder.putAll(properties);
+      }
+      return this;
+    }
+
+    @Override
+    public Catalog.TableBuilder withProperty(String key, String value) {
+      propertiesBuilder.put(key, value);
+      return this;
+    }
+
+    @Override
+    public Table create() {
+      TableOperations ops = newTableOps(location);
+      if (ops.current() != null) {
+        throw new AlreadyExistsException("Table already exists at location: %s", location);
+      }
+
+      Map<String, String> properties = propertiesBuilder.build();
+      TableMetadata metadata = tableMetadata(schema, spec, sortOrder, properties, location);
+      ops.commit(null, metadata);
+      return new BaseTable(ops, location);
+    }
+
+    @Override
+    public Transaction createTransaction() {
+      TableOperations ops = newTableOps(location);
+      if (ops.current() != null) {
+        throw new AlreadyExistsException("Table already exists: %s", location);
+      }
+
+      Map<String, String> properties = propertiesBuilder.build();
+      TableMetadata metadata = tableMetadata(schema, spec, null, properties, location);
+      return Transactions.createTableTransaction(location, ops, metadata);
+    }
+
+    @Override
+    public Transaction replaceTransaction() {
+      return newReplaceTableTransaction(false);
+    }
+
+    @Override
+    public Transaction createOrReplaceTransaction() {
+      return newReplaceTableTransaction(true);
+    }
+
+    private Transaction newReplaceTableTransaction(boolean orCreate) {
+      TableOperations ops = newTableOps(location);
+      if (!orCreate && ops.current() == null) {
+        throw new NoSuchTableException("No such table: %s", location);
+      }
+
+      Map<String, String> properties = propertiesBuilder.build();
+      TableMetadata metadata;
+      if (ops.current() != null) {
+        metadata = ops.current().buildReplacement(schema, spec, SortOrder.unsorted(), location, properties);
+      } else {
+        metadata = tableMetadata(schema, spec, null, properties, location);

Review comment:
       @rymurr  Here we use the `SortOrder.unsorted` or `null` to set the sort order rather than the `sortOrder` set from [here](https://github.com/apache/iceberg/pull/1843/files#diff-633b0a39dfee3393aee1717d9d83cd7e689d8672b24da82f809d8c21a933622dR271) ?  Is it correct ?  IMO,  we've ignored the user-provied sort order ? 




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org



---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org