You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@xtable.apache.org by as...@apache.org on 2024/03/27 20:40:37 UTC
(incubator-xtable) 01/01: Refactor, rename schema and snapshot classes with One in prefix
This is an automated email from the ASF dual-hosted git repository.
ashvin pushed a commit to branch 394-substitute-prefix-in-classnames
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit c279efb831e90e4aebc317f6460f28faecda6b85
Author: Ashvin Agrawal <as...@apache.org>
AuthorDate: Wed Mar 27 13:39:19 2024 -0700
Refactor, rename schema and snapshot classes with One in prefix
---
.../java/org/apache/xtable/model/OneTable.java | 4 +-
.../schema/{OneSchema.java => InternalSchema.java} | 11 +-
.../org/apache/xtable/model/schema/OneField.java | 2 +-
.../apache/xtable/model/schema/SchemaCatalog.java | 4 +-
.../java/org/apache/xtable/model/stat/Range.java | 11 +-
.../xtable/spi/extractor/SchemaExtractor.java | 4 +-
.../extractor/SourcePartitionSpecExtractor.java | 4 +-
.../org/apache/xtable/spi/sync/TargetClient.java | 4 +-
.../xtable/spi/sync/TestTableFormatSync.java | 4 +-
.../apache/xtable/avro/AvroSchemaConverter.java | 176 ++++++++--------
.../java/org/apache/xtable/delta/DeltaClient.java | 12 +-
.../xtable/delta/DeltaDataFileExtractor.java | 7 +-
.../delta/DeltaDataFileUpdatesExtractor.java | 13 +-
.../xtable/delta/DeltaPartitionExtractor.java | 38 ++--
.../apache/xtable/delta/DeltaSchemaExtractor.java | 46 +++--
.../org/apache/xtable/delta/DeltaSourceClient.java | 6 +-
.../apache/xtable/delta/DeltaStatsExtractor.java | 6 +-
.../apache/xtable/delta/DeltaTableExtractor.java | 4 +-
.../apache/xtable/delta/DeltaValueConverter.java | 26 +--
.../ConfigurationBasedPartitionSpecExtractor.java | 4 +-
.../apache/xtable/hudi/HudiFileStatsExtractor.java | 4 +-
.../xtable/hudi/HudiSchemaCatalogExtractor.java | 4 +-
.../apache/xtable/hudi/HudiSchemaExtractor.java | 10 +-
.../org/apache/xtable/hudi/HudiTableExtractor.java | 6 +-
.../org/apache/xtable/hudi/HudiTargetClient.java | 8 +-
.../org/apache/xtable/iceberg/IcebergClient.java | 4 +-
.../xtable/iceberg/IcebergDataFileExtractor.java | 6 +-
.../iceberg/IcebergPartitionSpecExtractor.java | 4 +-
.../xtable/iceberg/IcebergSchemaExtractor.java | 50 ++---
.../apache/xtable/iceberg/IcebergSourceClient.java | 8 +-
.../apache/xtable/schema/SchemaFieldFinder.java | 10 +-
.../xtable/avro/TestAvroSchemaConverter.java | 213 ++++++++++----------
.../apache/xtable/delta/ITDeltaSourceClient.java | 34 ++--
.../xtable/delta/TestDeltaPartitionExtractor.java | 81 +++++---
.../xtable/delta/TestDeltaSchemaExtractor.java | 221 ++++++++++----------
.../xtable/delta/TestDeltaStatsExtractor.java | 18 +-
.../org/apache/xtable/delta/TestDeltaSync.java | 44 ++--
.../xtable/delta/TestDeltaValueConverter.java | 39 ++--
.../org/apache/xtable/hudi/ITHudiTargetClient.java | 10 +-
.../xtable/hudi/TestBaseFileUpdatesExtractor.java | 4 +-
.../xtable/hudi/TestHudiFileStatsExtractor.java | 39 ++--
.../hudi/TestHudiPartitionValuesExtractor.java | 52 +++--
.../xtable/hudi/TestHudiSchemaExtractor.java | 10 +-
.../apache/xtable/hudi/TestHudiTableManager.java | 6 +-
.../apache/xtable/hudi/TestHudiTargetClient.java | 14 +-
.../iceberg/TestIcebergColumnStatsConverter.java | 30 +--
.../iceberg/TestIcebergPartitionSpecExtractor.java | 32 +--
.../TestIcebergPartitionValueConverter.java | 4 +-
.../xtable/iceberg/TestIcebergSchemaExtractor.java | 224 +++++++++++----------
.../xtable/iceberg/TestIcebergSourceClient.java | 8 +-
.../org/apache/xtable/iceberg/TestIcebergSync.java | 107 +++++-----
.../apache/xtable/testutil/ColumnStatMapUtil.java | 56 +++---
52 files changed, 917 insertions(+), 829 deletions(-)
diff --git a/api/src/main/java/org/apache/xtable/model/OneTable.java b/api/src/main/java/org/apache/xtable/model/OneTable.java
index 207e1d9c..f7cbe7f4 100644
--- a/api/src/main/java/org/apache/xtable/model/OneTable.java
+++ b/api/src/main/java/org/apache/xtable/model/OneTable.java
@@ -24,8 +24,8 @@ import java.util.List;
import lombok.Builder;
import lombok.Value;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataLayoutStrategy;
/**
@@ -41,7 +41,7 @@ public class OneTable {
// table format the table currently has data in
String tableFormat;
// Schema to use for reading the table
- OneSchema readSchema;
+ InternalSchema readSchema;
// Data layout strategy
DataLayoutStrategy layoutStrategy;
// Base path
diff --git a/api/src/main/java/org/apache/xtable/model/schema/OneSchema.java b/api/src/main/java/org/apache/xtable/model/schema/InternalSchema.java
similarity index 90%
rename from api/src/main/java/org/apache/xtable/model/schema/OneSchema.java
rename to api/src/main/java/org/apache/xtable/model/schema/InternalSchema.java
index fb0b5117..5af5a178 100644
--- a/api/src/main/java/org/apache/xtable/model/schema/OneSchema.java
+++ b/api/src/main/java/org/apache/xtable/model/schema/InternalSchema.java
@@ -31,8 +31,9 @@ import lombok.Getter;
import lombok.ToString;
/**
- * OneSchema represents a schema which could be a composite containing multiple {@link OneField} or
- * one of the primitives. Any level of schema hierarchy can be represented in this model.
+ * InternalSchema represents a schema which could be a composite containing multiple {@link
+ * OneField} or one of the primitives. Any level of schema hierarchy can be represented in this
+ * model.
*
* @since 0.1
*/
@@ -40,7 +41,7 @@ import lombok.ToString;
@Builder(toBuilder = true)
@EqualsAndHashCode
@ToString
-public class OneSchema {
+public class InternalSchema {
// The name of this schema
private final String name;
// The data type of this schema
@@ -60,7 +61,7 @@ public class OneSchema {
@Builder.Default List<OneField> recordKeyFields = Collections.emptyList();
private final Map<MetadataKey, Object> metadata;
- public static OneSchemaBuilder builderFrom(OneSchema field) {
+ public static InternalSchemaBuilder builderFrom(InternalSchema field) {
return field.toBuilder();
}
@@ -79,7 +80,7 @@ public class OneSchema {
/**
* Performs a level-order traversal of the schema and returns a list of all fields. Use this
- * method to get a list that includes nested fields. Use {@link OneSchema#getFields()} when
+ * method to get a list that includes nested fields. Use {@link InternalSchema#getFields()} when
* fetching the top level fields.
*
* @return list of all fields in the schema
diff --git a/api/src/main/java/org/apache/xtable/model/schema/OneField.java b/api/src/main/java/org/apache/xtable/model/schema/OneField.java
index 409e5ee6..3389b840 100644
--- a/api/src/main/java/org/apache/xtable/model/schema/OneField.java
+++ b/api/src/main/java/org/apache/xtable/model/schema/OneField.java
@@ -37,7 +37,7 @@ public class OneField {
// A dot separated path to the parent of this field. Null if this is a top level field.
String parentPath;
// Schema for the field
- OneSchema schema;
+ InternalSchema schema;
// Default value for the field
Object defaultValue;
// The id field for the field. This is used to identify the field in the schema even after
diff --git a/api/src/main/java/org/apache/xtable/model/schema/SchemaCatalog.java b/api/src/main/java/org/apache/xtable/model/schema/SchemaCatalog.java
index b9729580..64261425 100644
--- a/api/src/main/java/org/apache/xtable/model/schema/SchemaCatalog.java
+++ b/api/src/main/java/org/apache/xtable/model/schema/SchemaCatalog.java
@@ -33,9 +33,9 @@ import lombok.Value;
@Value
@Builder
public class SchemaCatalog {
- Map<SchemaVersion, OneSchema> schemas;
+ Map<SchemaVersion, InternalSchema> schemas;
- public SchemaCatalog(Map<SchemaVersion, OneSchema> schemas) {
+ public SchemaCatalog(Map<SchemaVersion, InternalSchema> schemas) {
this.schemas = schemas;
}
}
diff --git a/api/src/main/java/org/apache/xtable/model/stat/Range.java b/api/src/main/java/org/apache/xtable/model/stat/Range.java
index 80e56424..0b2ae611 100644
--- a/api/src/main/java/org/apache/xtable/model/stat/Range.java
+++ b/api/src/main/java/org/apache/xtable/model/stat/Range.java
@@ -22,7 +22,7 @@ import java.util.Objects;
import lombok.Value;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.InternalDataFile;
@@ -42,11 +42,12 @@ import org.apache.xtable.model.storage.InternalDataFile;
*
* <ul>
* <li>{@link OneType#TIMESTAMP} will be stored as a long represent millis or micros since epoch
- * depending on the {@link OneSchema.MetadataKey#TIMESTAMP_PRECISION}'s value ({@link
- * OneSchema.MetadataValue#MICROS} or {@link OneSchema.MetadataValue#MILLIS})
+ * depending on the {@link InternalSchema.MetadataKey#TIMESTAMP_PRECISION}'s value ({@link
+ * InternalSchema.MetadataValue#MICROS} or {@link InternalSchema.MetadataValue#MILLIS})
* <li>{@link OneType#TIMESTAMP_NTZ} will be stored as a long represent millis or micros since
- * epoch depending on the {@link OneSchema.MetadataKey#TIMESTAMP_PRECISION}'s value ({@link
- * OneSchema.MetadataValue#MICROS} or {@link OneSchema.MetadataValue#MILLIS})
+ * epoch depending on the {@link InternalSchema.MetadataKey#TIMESTAMP_PRECISION}'s value
+ * ({@link InternalSchema.MetadataValue#MICROS} or {@link
+ * InternalSchema.MetadataValue#MILLIS})
* <li>{@link OneType#DATE} will be stored as an integer representing days since epoch
* <li>{@link OneType#ENUM} will be stored as a string
* <li>{@link OneType#FIXED} will be stored as a {@link java.nio.ByteBuffer}
diff --git a/api/src/main/java/org/apache/xtable/spi/extractor/SchemaExtractor.java b/api/src/main/java/org/apache/xtable/spi/extractor/SchemaExtractor.java
index 37088c06..15a4bf05 100644
--- a/api/src/main/java/org/apache/xtable/spi/extractor/SchemaExtractor.java
+++ b/api/src/main/java/org/apache/xtable/spi/extractor/SchemaExtractor.java
@@ -18,7 +18,7 @@
package org.apache.xtable.spi.extractor;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
/**
* Interface to implement to extract canonical schema from a table format
@@ -26,5 +26,5 @@ import org.apache.xtable.model.schema.OneSchema;
* @param <CLIENT> Extracts canonical schema from table client.
*/
public interface SchemaExtractor<CLIENT> {
- OneSchema schema(CLIENT client);
+ InternalSchema schema(CLIENT client);
}
diff --git a/api/src/main/java/org/apache/xtable/spi/extractor/SourcePartitionSpecExtractor.java b/api/src/main/java/org/apache/xtable/spi/extractor/SourcePartitionSpecExtractor.java
index f7250694..f741859d 100644
--- a/api/src/main/java/org/apache/xtable/spi/extractor/SourcePartitionSpecExtractor.java
+++ b/api/src/main/java/org/apache/xtable/spi/extractor/SourcePartitionSpecExtractor.java
@@ -20,13 +20,13 @@ package org.apache.xtable.spi.extractor;
import java.util.List;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
/**
* Interface to implement to extract partition fields from given canonical schema. Refer @see
* org.apache.xtable.hudi.ConfigurationBasedPartitionSpecExtractor for an example implementation.
*/
public interface SourcePartitionSpecExtractor {
- List<OnePartitionField> spec(OneSchema tableSchema);
+ List<OnePartitionField> spec(InternalSchema tableSchema);
}
diff --git a/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java b/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
index db216246..76df4bd3 100644
--- a/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
+++ b/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
@@ -26,8 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.xtable.client.PerTableConfig;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.OneFileGroup;
@@ -39,7 +39,7 @@ public interface TargetClient {
*
* @param schema the current schema
*/
- void syncSchema(OneSchema schema);
+ void syncSchema(InternalSchema schema);
/**
* Syncs the current partition spec.
diff --git a/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java b/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
index 98363b3f..bd9230b8 100644
--- a/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
+++ b/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
@@ -44,9 +44,9 @@ import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
@@ -345,7 +345,7 @@ public class TestTableFormatSync {
private static OneTable getTableState(int id) {
return OneTable.builder()
.tableFormat(TableFormat.HUDI)
- .readSchema(OneSchema.builder().name(String.format("test_schema_%d", id)).build())
+ .readSchema(InternalSchema.builder().name(String.format("test_schema_%d", id)).build())
.partitioningFields(
Collections.singletonList(
OnePartitionField.builder()
diff --git a/core/src/main/java/org/apache/xtable/avro/AvroSchemaConverter.java b/core/src/main/java/org/apache/xtable/avro/AvroSchemaConverter.java
index 8058b7a0..bd171cab 100644
--- a/core/src/main/java/org/apache/xtable/avro/AvroSchemaConverter.java
+++ b/core/src/main/java/org/apache/xtable/avro/AvroSchemaConverter.java
@@ -40,13 +40,13 @@ import org.apache.xtable.exception.SchemaExtractorException;
import org.apache.xtable.exception.UnsupportedSchemaTypeException;
import org.apache.xtable.hudi.idtracking.IdTracker;
import org.apache.xtable.hudi.idtracking.models.IdMapping;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.schema.SchemaUtils;
/**
- * Class that converts Avro Schema {@link Schema} to Canonical Schema {@link OneSchema} and
+ * Class that converts Avro Schema {@link Schema} to Canonical Schema {@link InternalSchema} and
* vice-versa. This conversion is fully reversible and there is a strict 1 to 1 mapping between avro
* data types and canonical data types.
*/
@@ -57,7 +57,7 @@ public class AvroSchemaConverter {
OneField.builder()
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.STRING)
.isNullable(false)
@@ -73,7 +73,7 @@ public class AvroSchemaConverter {
return INSTANCE;
}
- public OneSchema toOneSchema(Schema schema) {
+ public InternalSchema toInternalSchema(Schema schema) {
Map<String, IdMapping> fieldNameToIdMapping =
IdTracker.getInstance()
.getIdTracking(schema)
@@ -82,11 +82,11 @@ public class AvroSchemaConverter {
idTracking.getIdMappings().stream()
.collect(Collectors.toMap(IdMapping::getName, Function.identity())))
.orElse(Collections.emptyMap());
- return toOneSchema(schema, null, fieldNameToIdMapping);
+ return toInternalSchema(schema, null, fieldNameToIdMapping);
}
/**
- * Converts the Avro {@link Schema} to {@link OneSchema}.
+ * Converts the Avro {@link Schema} to {@link InternalSchema}.
*
* @param schema The schema being converted
* @param parentPath If this schema is nested within another, this will be a dot separated string
@@ -95,11 +95,11 @@ public class AvroSchemaConverter {
* source schema. If source schema does not contain IdMappings, map will be empty.
* @return a converted schema
*/
- private OneSchema toOneSchema(
+ private InternalSchema toInternalSchema(
Schema schema, String parentPath, Map<String, IdMapping> fieldNameToIdMapping) {
// TODO - Does not handle recursion in Avro schema
OneType newDataType;
- Map<OneSchema.MetadataKey, Object> metadata = new HashMap<>();
+ Map<InternalSchema.MetadataKey, Object> metadata = new HashMap<>();
switch (schema.getType()) {
case INT:
LogicalType logicalType = schema.getLogicalType();
@@ -120,18 +120,19 @@ public class AvroSchemaConverter {
logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.Decimal) {
metadata.put(
- OneSchema.MetadataKey.DECIMAL_PRECISION,
+ InternalSchema.MetadataKey.DECIMAL_PRECISION,
((LogicalTypes.Decimal) logicalType).getPrecision());
metadata.put(
- OneSchema.MetadataKey.DECIMAL_SCALE, ((LogicalTypes.Decimal) logicalType).getScale());
+ InternalSchema.MetadataKey.DECIMAL_SCALE,
+ ((LogicalTypes.Decimal) logicalType).getScale());
if (schema.getType() == Schema.Type.FIXED) {
- metadata.put(OneSchema.MetadataKey.FIXED_BYTES_SIZE, schema.getFixedSize());
+ metadata.put(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, schema.getFixedSize());
}
newDataType = OneType.DECIMAL;
break;
}
if (schema.getType() == Schema.Type.FIXED) {
- metadata.put(OneSchema.MetadataKey.FIXED_BYTES_SIZE, schema.getFixedSize());
+ metadata.put(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, schema.getFixedSize());
newDataType = OneType.FIXED;
} else {
newDataType = OneType.BYTES;
@@ -147,22 +148,26 @@ public class AvroSchemaConverter {
logicalType = schema.getLogicalType();
if (logicalType instanceof LogicalTypes.TimestampMillis) {
newDataType = OneType.TIMESTAMP;
- metadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
+ metadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
} else if (logicalType instanceof LogicalTypes.TimestampMicros) {
newDataType = OneType.TIMESTAMP;
- metadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
+ metadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
} else if (logicalType instanceof LogicalTypes.LocalTimestampMillis) {
newDataType = OneType.TIMESTAMP_NTZ;
- metadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
+ metadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
} else if (logicalType instanceof LogicalTypes.LocalTimestampMicros) {
newDataType = OneType.TIMESTAMP_NTZ;
- metadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
+ metadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
} else {
newDataType = OneType.LONG;
}
break;
case ENUM:
- metadata.put(OneSchema.MetadataKey.ENUM_VALUES, schema.getEnumSymbols());
+ metadata.put(InternalSchema.MetadataKey.ENUM_VALUES, schema.getEnumSymbols());
newDataType = OneType.ENUM;
break;
case NULL:
@@ -172,8 +177,8 @@ public class AvroSchemaConverter {
List<OneField> subFields = new ArrayList<>(schema.getFields().size());
for (Schema.Field avroField : schema.getFields()) {
IdMapping idMapping = fieldNameToIdMapping.get(avroField.name());
- OneSchema subFieldSchema =
- toOneSchema(
+ InternalSchema subFieldSchema =
+ toInternalSchema(
avroField.schema(),
SchemaUtils.getFullyQualifiedPath(parentPath, avroField.name()),
getChildIdMap(idMapping));
@@ -187,7 +192,7 @@ public class AvroSchemaConverter {
.fieldId(idMapping == null ? null : idMapping.getId())
.build());
}
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(schema.getName())
.comment(schema.getDoc())
.dataType(OneType.RECORD)
@@ -196,8 +201,8 @@ public class AvroSchemaConverter {
.build();
case ARRAY:
IdMapping elementMapping = fieldNameToIdMapping.get(ELEMENT);
- OneSchema elementSchema =
- toOneSchema(
+ InternalSchema elementSchema =
+ toInternalSchema(
schema.getElementType(),
SchemaUtils.getFullyQualifiedPath(
parentPath, OneField.Constants.ARRAY_ELEMENT_FIELD_NAME),
@@ -209,7 +214,7 @@ public class AvroSchemaConverter {
.schema(elementSchema)
.fieldId(elementMapping == null ? null : elementMapping.getId())
.build();
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(schema.getName())
.dataType(OneType.LIST)
.comment(schema.getDoc())
@@ -219,8 +224,8 @@ public class AvroSchemaConverter {
case MAP:
IdMapping keyMapping = fieldNameToIdMapping.get(KEY);
IdMapping valueMapping = fieldNameToIdMapping.get(VALUE);
- OneSchema valueSchema =
- toOneSchema(
+ InternalSchema valueSchema =
+ toInternalSchema(
schema.getValueType(),
SchemaUtils.getFullyQualifiedPath(
parentPath, OneField.Constants.MAP_VALUE_FIELD_NAME),
@@ -232,7 +237,7 @@ public class AvroSchemaConverter {
.schema(valueSchema)
.fieldId(valueMapping == null ? null : valueMapping.getId())
.build();
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(schema.getName())
.dataType(OneType.MAP)
.comment(schema.getDoc())
@@ -254,11 +259,12 @@ public class AvroSchemaConverter {
.filter(t -> t.getType() != Schema.Type.NULL)
.collect(Collectors.toList());
if (remainingSchemas.size() == 1) {
- OneSchema restSchema =
- toOneSchema(remainingSchemas.get(0), parentPath, fieldNameToIdMapping);
- return OneSchema.builderFrom(restSchema).isNullable(true).build();
+ InternalSchema restSchema =
+ toInternalSchema(remainingSchemas.get(0), parentPath, fieldNameToIdMapping);
+ return InternalSchema.builderFrom(restSchema).isNullable(true).build();
} else {
- return OneSchema.builderFrom(toOneSchema(Schema.createUnion(remainingSchemas)))
+ return InternalSchema.builderFrom(
+ toInternalSchema(Schema.createUnion(remainingSchemas)))
.isNullable(true)
.build();
}
@@ -270,7 +276,7 @@ public class AvroSchemaConverter {
throw new UnsupportedSchemaTypeException(
String.format("Unsupported schema type %s", schema));
}
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(schema.getName())
.dataType(newDataType)
.comment(schema.getDoc())
@@ -294,137 +300,143 @@ public class AvroSchemaConverter {
}
/**
- * Converts the {@link OneSchema} to Avro {@link Schema}.
+ * Converts the {@link InternalSchema} to Avro {@link Schema}.
*
- * @param oneSchema internal schema representation
+ * @param internalSchema internal schema representation
* @return an Avro schema
*/
- public Schema fromOneSchema(OneSchema oneSchema) {
- return fromOneSchema(oneSchema, null);
+ public Schema fromInternalSchema(InternalSchema internalSchema) {
+ return fromInternalSchema(internalSchema, null);
}
/**
- * Internal method for converting the {@link OneSchema} to Avro {@link Schema}.
+ * Internal method for converting the {@link InternalSchema} to Avro {@link Schema}.
*
- * @param oneSchema internal schema representation
+ * @param internalSchema internal schema representation
* @param currentPath If this schema is nested within another, this will be a dot separated
* string. This is used for the avro namespace to guarantee unique names for nested records.
* @return an Avro schema
*/
- private Schema fromOneSchema(OneSchema oneSchema, String currentPath) {
- switch (oneSchema.getDataType()) {
+ private Schema fromInternalSchema(InternalSchema internalSchema, String currentPath) {
+ switch (internalSchema.getDataType()) {
case RECORD:
List<Schema.Field> fields =
- oneSchema.getFields().stream()
+ internalSchema.getFields().stream()
.map(
field ->
new Schema.Field(
field.getName(),
- fromOneSchema(
+ fromInternalSchema(
field.getSchema(),
SchemaUtils.getFullyQualifiedPath(currentPath, field.getName())),
field.getSchema().getComment(),
OneField.Constants.NULL_DEFAULT_VALUE == field.getDefaultValue()
? Schema.Field.NULL_VALUE
: field.getDefaultValue()))
- .collect(CustomCollectors.toList(oneSchema.getFields().size()));
+ .collect(CustomCollectors.toList(internalSchema.getFields().size()));
return finalizeSchema(
Schema.createRecord(
- oneSchema.getName(), oneSchema.getComment(), currentPath, false, fields),
- oneSchema);
+ internalSchema.getName(), internalSchema.getComment(), currentPath, false, fields),
+ internalSchema);
case BYTES:
- return finalizeSchema(Schema.create(Schema.Type.BYTES), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.BYTES), internalSchema);
case BOOLEAN:
- return finalizeSchema(Schema.create(Schema.Type.BOOLEAN), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.BOOLEAN), internalSchema);
case INT:
- return finalizeSchema(Schema.create(Schema.Type.INT), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.INT), internalSchema);
case LONG:
- return finalizeSchema(Schema.create(Schema.Type.LONG), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.LONG), internalSchema);
case STRING:
- return finalizeSchema(Schema.create(Schema.Type.STRING), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.STRING), internalSchema);
case FLOAT:
- return finalizeSchema(Schema.create(Schema.Type.FLOAT), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.FLOAT), internalSchema);
case DOUBLE:
- return finalizeSchema(Schema.create(Schema.Type.DOUBLE), oneSchema);
+ return finalizeSchema(Schema.create(Schema.Type.DOUBLE), internalSchema);
case ENUM:
return finalizeSchema(
Schema.createEnum(
- oneSchema.getName(),
- oneSchema.getComment(),
+ internalSchema.getName(),
+ internalSchema.getComment(),
null,
- (List<String>) oneSchema.getMetadata().get(OneSchema.MetadataKey.ENUM_VALUES),
+ (List<String>)
+ internalSchema.getMetadata().get(InternalSchema.MetadataKey.ENUM_VALUES),
null),
- oneSchema);
+ internalSchema);
case DATE:
return finalizeSchema(
- LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)), oneSchema);
+ LogicalTypes.date().addToSchema(Schema.create(Schema.Type.INT)), internalSchema);
case TIMESTAMP:
- if (oneSchema.getMetadata().get(OneSchema.MetadataKey.TIMESTAMP_PRECISION)
- == OneSchema.MetadataValue.MICROS) {
+ if (internalSchema.getMetadata().get(InternalSchema.MetadataKey.TIMESTAMP_PRECISION)
+ == InternalSchema.MetadataValue.MICROS) {
return finalizeSchema(
LogicalTypes.timestampMicros().addToSchema(Schema.create(Schema.Type.LONG)),
- oneSchema);
+ internalSchema);
} else {
return finalizeSchema(
LogicalTypes.timestampMillis().addToSchema(Schema.create(Schema.Type.LONG)),
- oneSchema);
+ internalSchema);
}
case TIMESTAMP_NTZ:
- if (oneSchema.getMetadata().get(OneSchema.MetadataKey.TIMESTAMP_PRECISION)
- == OneSchema.MetadataValue.MICROS) {
+ if (internalSchema.getMetadata().get(InternalSchema.MetadataKey.TIMESTAMP_PRECISION)
+ == InternalSchema.MetadataValue.MICROS) {
return finalizeSchema(
LogicalTypes.localTimestampMicros().addToSchema(Schema.create(Schema.Type.LONG)),
- oneSchema);
+ internalSchema);
} else {
return finalizeSchema(
LogicalTypes.localTimestampMillis().addToSchema(Schema.create(Schema.Type.LONG)),
- oneSchema);
+ internalSchema);
}
case LIST:
OneField elementField =
- oneSchema.getFields().stream()
+ internalSchema.getFields().stream()
.filter(
field -> OneField.Constants.ARRAY_ELEMENT_FIELD_NAME.equals(field.getName()))
.findFirst()
.orElseThrow(() -> new SchemaExtractorException("Invalid array schema"));
return finalizeSchema(
- Schema.createArray(fromOneSchema(elementField.getSchema(), elementField.getPath())),
- oneSchema);
+ Schema.createArray(
+ fromInternalSchema(elementField.getSchema(), elementField.getPath())),
+ internalSchema);
case MAP:
OneField valueField =
- oneSchema.getFields().stream()
+ internalSchema.getFields().stream()
.filter(field -> OneField.Constants.MAP_VALUE_FIELD_NAME.equals(field.getName()))
.findFirst()
.orElseThrow(() -> new SchemaExtractorException("Invalid map schema"));
return finalizeSchema(
- Schema.createMap(fromOneSchema(valueField.getSchema(), valueField.getPath())),
- oneSchema);
+ Schema.createMap(fromInternalSchema(valueField.getSchema(), valueField.getPath())),
+ internalSchema);
case DECIMAL:
- int precision = (int) oneSchema.getMetadata().get(OneSchema.MetadataKey.DECIMAL_PRECISION);
- int scale = (int) oneSchema.getMetadata().get(OneSchema.MetadataKey.DECIMAL_SCALE);
+ int precision =
+ (int) internalSchema.getMetadata().get(InternalSchema.MetadataKey.DECIMAL_PRECISION);
+ int scale =
+ (int) internalSchema.getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE);
Integer size =
- (Integer) oneSchema.getMetadata().get(OneSchema.MetadataKey.FIXED_BYTES_SIZE);
+ (Integer) internalSchema.getMetadata().get(InternalSchema.MetadataKey.FIXED_BYTES_SIZE);
if (size == null) {
return finalizeSchema(
LogicalTypes.decimal(precision, scale).addToSchema(Schema.create(Schema.Type.BYTES)),
- oneSchema);
+ internalSchema);
} else {
return finalizeSchema(
LogicalTypes.decimal(precision, scale)
.addToSchema(
- Schema.createFixed(oneSchema.getName(), oneSchema.getComment(), null, size)),
- oneSchema);
+ Schema.createFixed(
+ internalSchema.getName(), internalSchema.getComment(), null, size)),
+ internalSchema);
}
case FIXED:
Integer fixedSize =
- (Integer) oneSchema.getMetadata().get(OneSchema.MetadataKey.FIXED_BYTES_SIZE);
+ (Integer) internalSchema.getMetadata().get(InternalSchema.MetadataKey.FIXED_BYTES_SIZE);
return finalizeSchema(
- Schema.createFixed(oneSchema.getName(), oneSchema.getComment(), null, fixedSize),
- oneSchema);
+ Schema.createFixed(
+ internalSchema.getName(), internalSchema.getComment(), null, fixedSize),
+ internalSchema);
default:
throw new UnsupportedSchemaTypeException(
- "Encountered unhandled type during OneSchema to Avro conversion: "
- + oneSchema.getDataType());
+ "Encountered unhandled type during InternalSchema to Avro conversion: "
+ + internalSchema.getDataType());
}
}
@@ -434,7 +446,7 @@ public class AvroSchemaConverter {
.orElse(field.getName());
}
- private static Schema finalizeSchema(Schema targetSchema, OneSchema inputSchema) {
+ private static Schema finalizeSchema(Schema targetSchema, InternalSchema inputSchema) {
if (inputSchema.isNullable()) {
return Schema.createUnion(Schema.create(Schema.Type.NULL), targetSchema);
}
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaClient.java b/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
index cee8568b..5bd3f115 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
@@ -58,8 +58,8 @@ import org.apache.xtable.client.PerTableConfig;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.OneFileGroup;
import org.apache.xtable.model.storage.TableFormat;
@@ -154,7 +154,7 @@ public class DeltaClient implements TargetClient {
}
@Override
- public void syncSchema(OneSchema schema) {
+ public void syncSchema(InternalSchema schema) {
transactionState.setLatestSchema(schema);
}
@@ -222,7 +222,7 @@ public class DeltaClient implements TargetClient {
@Getter private final List<String> partitionColumns;
private final String tableName;
@Getter private StructType latestSchema;
- @Getter private OneSchema latestSchemaInternal;
+ @Getter private InternalSchema latestSchemaInternal;
@Setter private OneTableMetadata metadata;
@Setter private Seq<Action> actions;
@@ -239,12 +239,12 @@ public class DeltaClient implements TargetClient {
private void addColumn(StructField field) {
latestSchema = latestSchema.add(field);
- latestSchemaInternal = schemaExtractor.toOneSchema(latestSchema);
+ latestSchemaInternal = schemaExtractor.toInternalSchema(latestSchema);
}
- private void setLatestSchema(OneSchema schema) {
+ private void setLatestSchema(InternalSchema schema) {
this.latestSchemaInternal = schema;
- this.latestSchema = schemaExtractor.fromOneSchema(schema);
+ this.latestSchema = schemaExtractor.fromInternalSchema(schema);
}
private void commitTransaction() {
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileExtractor.java
index 95507826..a9705ed9 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileExtractor.java
@@ -25,9 +25,9 @@ import lombok.Builder;
import org.apache.spark.sql.delta.Snapshot;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
import org.apache.xtable.spi.extractor.DataFileIterator;
@@ -50,7 +50,7 @@ public class DeltaDataFileExtractor {
*
* @return Delta table file iterator
*/
- public DataFileIterator iterator(Snapshot deltaSnapshot, OneSchema schema) {
+ public DataFileIterator iterator(Snapshot deltaSnapshot, InternalSchema schema) {
return new DeltaDataFileIterator(deltaSnapshot, schema, true);
}
@@ -60,7 +60,8 @@ public class DeltaDataFileExtractor {
private final List<OnePartitionField> partitionFields;
private final Iterator<InternalDataFile> dataFilesIterator;
- private DeltaDataFileIterator(Snapshot snapshot, OneSchema schema, boolean includeColumnStats) {
+ private DeltaDataFileIterator(
+ Snapshot snapshot, InternalSchema schema, boolean includeColumnStats) {
this.fileFormat =
actionsConverter.convertToOneTableFileFormat(snapshot.metadata().format().provider());
this.fields = schema.getFields();
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
index 5d7de074..f2ad006f 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
@@ -37,7 +37,7 @@ import scala.collection.Seq;
import com.fasterxml.jackson.core.JsonProcessingException;
import org.apache.xtable.collectors.CustomCollectors;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FilesDiff;
@@ -59,7 +59,7 @@ public class DeltaDataFileUpdatesExtractor {
DeltaDataFileExtractor.builder().build();
public Seq<Action> applySnapshot(
- DeltaLog deltaLog, List<OneFileGroup> partitionedDataFiles, OneSchema tableSchema) {
+ DeltaLog deltaLog, List<OneFileGroup> partitionedDataFiles, InternalSchema tableSchema) {
// all files in the current delta snapshot are potential candidates for remove actions, i.e. if
// the file is not present in the new snapshot (addedFiles) then the file is considered removed
@@ -80,7 +80,7 @@ public class DeltaDataFileUpdatesExtractor {
}
public Seq<Action> applyDiff(
- DataFilesDiff dataFilesDiff, OneSchema tableSchema, String tableBasePath) {
+ DataFilesDiff dataFilesDiff, InternalSchema tableSchema, String tableBasePath) {
List<Action> removeActions =
dataFilesDiff.getFilesRemoved().stream()
.flatMap(dFile -> createAddFileAction(dFile, tableSchema, tableBasePath))
@@ -92,7 +92,7 @@ public class DeltaDataFileUpdatesExtractor {
private Seq<Action> applyDiff(
Set<InternalDataFile> filesAdded,
Collection<Action> removeFileActions,
- OneSchema tableSchema,
+ InternalSchema tableSchema,
String tableBasePath) {
Stream<Action> addActions =
filesAdded.stream()
@@ -105,7 +105,7 @@ public class DeltaDataFileUpdatesExtractor {
}
private Stream<AddFile> createAddFileAction(
- InternalDataFile dataFile, OneSchema schema, String tableBasePath) {
+ InternalDataFile dataFile, InternalSchema schema, String tableBasePath) {
return Stream.of(
new AddFile(
// Delta Lake supports relative and absolute paths in theory but relative paths seem
@@ -120,7 +120,8 @@ public class DeltaDataFileUpdatesExtractor {
null));
}
- private String getColumnStats(OneSchema schema, long recordCount, List<ColumnStat> columnStats) {
+ private String getColumnStats(
+ InternalSchema schema, long recordCount, List<ColumnStat> columnStats) {
try {
return deltaStatsExtractor.convertStatsToDeltaFormat(schema, recordCount, columnStats);
} catch (JsonProcessingException e) {
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
index 31eb0752..94e2e999 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaPartitionExtractor.java
@@ -51,8 +51,8 @@ import com.google.common.collect.Iterators;
import com.google.common.collect.PeekingIterator;
import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.PartitionValue;
import org.apache.xtable.model.stat.Range;
@@ -93,20 +93,20 @@ public class DeltaPartitionExtractor {
/**
* Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
* Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
- * deltaLog = DeltaLog.forTable(spark, deltaTablePath); OneSchema oneSchema =
- * DeltaSchemaExtractor.getInstance().toOneSchema(deltaLog.snapshot().schema()); StructType
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
* partitionSchema = deltaLog.metadata().partitionSchema();
*
- * @param oneSchema canonical representation of the schema.
+ * @param internalSchema canonical representation of the schema.
* @param partitionSchema partition schema of the delta table.
* @return list of canonical representation of the partition fields
*/
public List<OnePartitionField> convertFromDeltaPartitionFormat(
- OneSchema oneSchema, StructType partitionSchema) {
+ InternalSchema internalSchema, StructType partitionSchema) {
if (partitionSchema.isEmpty()) {
return Collections.emptyList();
}
- return getOnePartitionFields(partitionSchema, oneSchema);
+ return getOnePartitionFields(partitionSchema, internalSchema);
}
/**
@@ -116,7 +116,7 @@ public class DeltaPartitionExtractor {
* and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
*/
private List<OnePartitionField> getOnePartitionFields(
- StructType partitionSchema, OneSchema oneSchema) {
+ StructType partitionSchema, InternalSchema internalSchema) {
PeekingIterator<StructField> itr =
Iterators.peekingIterator(Arrays.stream(partitionSchema.fields()).iterator());
List<OnePartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().length);
@@ -127,7 +127,7 @@ public class DeltaPartitionExtractor {
OnePartitionField.builder()
.sourceField(
SchemaFieldFinder.getInstance()
- .findFieldByPath(oneSchema, currPartitionField.name()))
+ .findFieldByPath(internalSchema, currPartitionField.name()))
.transformType(PartitionTransformType.VALUE)
.build());
itr.next(); // consume the field.
@@ -141,13 +141,13 @@ public class DeltaPartitionExtractor {
if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
partitionFields.add(
getPartitionWithDateTransform(
- currPartitionField.name(), parsedGeneratedExpr, oneSchema));
+ currPartitionField.name(), parsedGeneratedExpr, internalSchema));
itr.next(); // consume the field.
} else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
== parsedGeneratedExpr.generatedExprType) {
partitionFields.add(
getPartitionWithDateFormatTransform(
- currPartitionField.name(), parsedGeneratedExpr, oneSchema));
+ currPartitionField.name(), parsedGeneratedExpr, internalSchema));
itr.next(); // consume the field.
} else {
// consume until we hit field with no generated expression or generated expression
@@ -171,7 +171,7 @@ public class DeltaPartitionExtractor {
}
}
partitionFields.add(
- getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, oneSchema));
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
}
}
}
@@ -179,7 +179,7 @@ public class DeltaPartitionExtractor {
}
private OnePartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
- List<ParsedGeneratedExpr> parsedGeneratedExprs, OneSchema oneSchema) {
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
if (parsedGeneratedExprs.size() > 4) {
throw new IllegalStateException("Invalid partition transform");
}
@@ -193,7 +193,7 @@ public class DeltaPartitionExtractor {
.collect(toList(parsedGeneratedExprs.size()));
return OnePartitionField.builder()
.sourceField(
- SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, transform.sourceColumn))
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
.partitionFieldNames(partitionColumns)
.transformType(
parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
@@ -203,22 +203,26 @@ public class DeltaPartitionExtractor {
// Cast has default format of yyyy-MM-dd.
private OnePartitionField getPartitionWithDateTransform(
- String partitionColumnName, ParsedGeneratedExpr parsedGeneratedExpr, OneSchema oneSchema) {
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
return OnePartitionField.builder()
.sourceField(
SchemaFieldFinder.getInstance()
- .findFieldByPath(oneSchema, parsedGeneratedExpr.sourceColumn))
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
.partitionFieldNames(Collections.singletonList(partitionColumnName))
.transformType(PartitionTransformType.DAY)
.build();
}
private OnePartitionField getPartitionWithDateFormatTransform(
- String partitionColumnName, ParsedGeneratedExpr parsedGeneratedExpr, OneSchema oneSchema) {
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
return OnePartitionField.builder()
.sourceField(
SchemaFieldFinder.getInstance()
- .findFieldByPath(oneSchema, parsedGeneratedExpr.sourceColumn))
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
.partitionFieldNames(Collections.singletonList(partitionColumnName))
.transformType(parsedGeneratedExpr.internalPartitionTransformType)
.build();
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
index 898a33c0..52b54385 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
@@ -39,8 +39,8 @@ import org.apache.spark.sql.types.StructType;
import org.apache.xtable.collectors.CustomCollectors;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.SchemaExtractorException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.schema.SchemaUtils;
@@ -64,9 +64,9 @@ public class DeltaSchemaExtractor {
return INSTANCE;
}
- public StructType fromOneSchema(OneSchema oneSchema) {
+ public StructType fromInternalSchema(InternalSchema internalSchema) {
StructField[] fields =
- oneSchema.getFields().stream()
+ internalSchema.getFields().stream()
.map(
field ->
new StructField(
@@ -103,11 +103,12 @@ public class DeltaSchemaExtractor {
return DataTypes.DoubleType;
case DECIMAL:
int precision =
- (int) field.getSchema().getMetadata().get(OneSchema.MetadataKey.DECIMAL_PRECISION);
- int scale = (int) field.getSchema().getMetadata().get(OneSchema.MetadataKey.DECIMAL_SCALE);
+ (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_PRECISION);
+ int scale =
+ (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE);
return DataTypes.createDecimalType(precision, scale);
case RECORD:
- return fromOneSchema(field.getSchema());
+ return fromInternalSchema(field.getSchema());
case MAP:
OneField key =
field.getSchema().getFields().stream()
@@ -138,13 +139,13 @@ public class DeltaSchemaExtractor {
}
}
- public OneSchema toOneSchema(StructType structType) {
- return toOneSchema(structType, null, false, null);
+ public InternalSchema toInternalSchema(StructType structType) {
+ return toInternalSchema(structType, null, false, null);
}
- private OneSchema toOneSchema(
+ private InternalSchema toInternalSchema(
DataType dataType, String parentPath, boolean nullable, String comment) {
- Map<OneSchema.MetadataKey, Object> metadata = null;
+ Map<InternalSchema.MetadataKey, Object> metadata = null;
List<OneField> fields = null;
OneType type;
String typeName = dataType.typeName();
@@ -181,7 +182,8 @@ public class DeltaSchemaExtractor {
// Timestamps in Delta are microsecond precision by default
metadata =
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS);
break;
case "struct":
StructType structType = (StructType) dataType;
@@ -200,8 +202,8 @@ public class DeltaSchemaExtractor {
: null;
String fieldComment =
field.getComment().isDefined() ? field.getComment().get() : null;
- OneSchema schema =
- toOneSchema(
+ InternalSchema schema =
+ toInternalSchema(
field.dataType(),
SchemaUtils.getFullyQualifiedPath(parentPath, field.name()),
field.nullable(),
@@ -221,14 +223,14 @@ public class DeltaSchemaExtractor {
case "decimal":
DecimalType decimalType = (DecimalType) dataType;
metadata = new HashMap<>(2, 1.0f);
- metadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, decimalType.precision());
- metadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, decimalType.scale());
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, decimalType.precision());
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, decimalType.scale());
type = OneType.DECIMAL;
break;
case "array":
ArrayType arrayType = (ArrayType) dataType;
- OneSchema elementSchema =
- toOneSchema(
+ InternalSchema elementSchema =
+ toInternalSchema(
arrayType.elementType(),
SchemaUtils.getFullyQualifiedPath(
parentPath, OneField.Constants.ARRAY_ELEMENT_FIELD_NAME),
@@ -245,8 +247,8 @@ public class DeltaSchemaExtractor {
break;
case "map":
MapType mapType = (MapType) dataType;
- OneSchema keySchema =
- toOneSchema(
+ InternalSchema keySchema =
+ toInternalSchema(
mapType.keyType(),
SchemaUtils.getFullyQualifiedPath(
parentPath, OneField.Constants.MAP_VALUE_FIELD_NAME),
@@ -258,8 +260,8 @@ public class DeltaSchemaExtractor {
.parentPath(parentPath)
.schema(keySchema)
.build();
- OneSchema valueSchema =
- toOneSchema(
+ InternalSchema valueSchema =
+ toInternalSchema(
mapType.valueType(),
SchemaUtils.getFullyQualifiedPath(
parentPath, OneField.Constants.MAP_VALUE_FIELD_NAME),
@@ -277,7 +279,7 @@ public class DeltaSchemaExtractor {
default:
throw new NotSupportedException("Unsupported type: " + dataType.typeName());
}
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(trimmedTypeName)
.dataType(type)
.comment(comment)
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java b/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
index caff2641..12baf2a0 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
@@ -50,7 +50,7 @@ import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.TableChange;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.model.storage.DataFilesDiff;
@@ -88,7 +88,7 @@ public class DeltaSourceClient implements SourceClient<Long> {
@Override
public SchemaCatalog getSchemaCatalog(OneTable table, Long version) {
// TODO: Does not support schema versions for now
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
schemas.put(schemaVersion, table.getReadSchema());
return SchemaCatalog.builder().schemas(schemas).build();
@@ -188,7 +188,7 @@ public class DeltaSourceClient implements SourceClient<Long> {
.build());
}
- private List<OneFileGroup> getInternalDataFiles(Snapshot snapshot, OneSchema schema) {
+ private List<OneFileGroup> getInternalDataFiles(Snapshot snapshot, InternalSchema schema) {
try (DataFileIterator fileIterator = dataFileExtractor.iterator(snapshot, schema)) {
List<InternalDataFile> dataFiles = new ArrayList<>();
fileIterator.forEachRemaining(dataFiles::add);
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaStatsExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaStatsExtractor.java
index be2fba12..7ffdedf2 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaStatsExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaStatsExtractor.java
@@ -43,8 +43,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.xtable.collectors.CustomCollectors;
import org.apache.xtable.exception.OneIOException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.Range;
@@ -79,7 +79,7 @@ public class DeltaStatsExtractor {
}
public String convertStatsToDeltaFormat(
- OneSchema schema, long numRecords, List<ColumnStat> columnStats)
+ InternalSchema schema, long numRecords, List<ColumnStat> columnStats)
throws JsonProcessingException {
DeltaStats.DeltaStatsBuilder deltaStatsBuilder = DeltaStats.builder();
deltaStatsBuilder.numRecords(numRecords);
@@ -100,7 +100,7 @@ public class DeltaStatsExtractor {
return MAPPER.writeValueAsString(deltaStats);
}
- private Set<String> getPathsFromStructSchemaForMinAndMaxStats(OneSchema schema) {
+ private Set<String> getPathsFromStructSchemaForMinAndMaxStats(InternalSchema schema) {
return schema.getAllFields().stream()
.filter(
field -> {
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
index 091f933d..f5a4a887 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
@@ -29,8 +29,8 @@ import org.apache.spark.sql.delta.Snapshot;
import scala.Option;
import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.TableFormat;
@@ -42,7 +42,7 @@ public class DeltaTableExtractor {
public OneTable table(DeltaLog deltaLog, String tableName, Long version) {
Snapshot snapshot = deltaLog.getSnapshotAt(version, Option.empty());
- OneSchema schema = schemaExtractor.toOneSchema(snapshot.metadata().schema());
+ InternalSchema schema = schemaExtractor.toInternalSchema(snapshot.metadata().schema());
List<OnePartitionField> partitionFields =
DeltaPartitionExtractor.getInstance()
.convertFromDeltaPartitionFormat(schema, snapshot.metadata().partitionSchema());
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaValueConverter.java b/core/src/main/java/org/apache/xtable/delta/DeltaValueConverter.java
index 09db128e..521e3b49 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaValueConverter.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaValueConverter.java
@@ -34,7 +34,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.OneIOException;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
@@ -53,7 +53,7 @@ public class DeltaValueConverter {
return dateFormat;
}
- public static Object convertFromDeltaColumnStatValue(Object value, OneSchema fieldSchema) {
+ public static Object convertFromDeltaColumnStatValue(Object value, InternalSchema fieldSchema) {
if (value == null) {
return null;
}
@@ -78,20 +78,21 @@ public class DeltaValueConverter {
throw new OneIOException("Unable to parse time from column stats", ex);
}
}
- OneSchema.MetadataValue timestampPrecision =
- (OneSchema.MetadataValue)
+ InternalSchema.MetadataValue timestampPrecision =
+ (InternalSchema.MetadataValue)
fieldSchema
.getMetadata()
.getOrDefault(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
- if (timestampPrecision == OneSchema.MetadataValue.MILLIS) {
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS);
+ if (timestampPrecision == InternalSchema.MetadataValue.MILLIS) {
return instant.toEpochMilli();
}
return TimeUnit.SECONDS.toMicros(instant.getEpochSecond())
+ TimeUnit.NANOSECONDS.toMicros(instant.getNano());
}
- public static Object convertToDeltaColumnStatValue(Object value, OneSchema fieldSchema) {
+ public static Object convertToDeltaColumnStatValue(Object value, InternalSchema fieldSchema) {
if (value == null) {
return null;
}
@@ -103,21 +104,22 @@ public class DeltaValueConverter {
if (fieldType == OneType.DATE) {
return LocalDate.ofEpochDay((int) value).toString();
}
- OneSchema.MetadataValue timestampPrecision =
- (OneSchema.MetadataValue)
+ InternalSchema.MetadataValue timestampPrecision =
+ (InternalSchema.MetadataValue)
fieldSchema
.getMetadata()
.getOrDefault(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MILLIS);
long millis =
- timestampPrecision == OneSchema.MetadataValue.MICROS
+ timestampPrecision == InternalSchema.MetadataValue.MICROS
? TimeUnit.MILLISECONDS.convert((Long) value, TimeUnit.MICROSECONDS)
: (long) value;
DateFormat dateFormat = getDateFormat(DATE_FORMAT_STR);
return dateFormat.format(Date.from(Instant.ofEpochMilli(millis)));
}
- private static boolean noConversionForSchema(OneSchema fieldSchema) {
+ private static boolean noConversionForSchema(InternalSchema fieldSchema) {
OneType fieldType = fieldSchema.getDataType();
return fieldType != OneType.DATE
&& fieldType != OneType.TIMESTAMP
diff --git a/core/src/main/java/org/apache/xtable/hudi/ConfigurationBasedPartitionSpecExtractor.java b/core/src/main/java/org/apache/xtable/hudi/ConfigurationBasedPartitionSpecExtractor.java
index e3bed5f2..3ca84604 100644
--- a/core/src/main/java/org/apache/xtable/hudi/ConfigurationBasedPartitionSpecExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/ConfigurationBasedPartitionSpecExtractor.java
@@ -25,9 +25,9 @@ import java.util.Map;
import lombok.AllArgsConstructor;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.schema.SchemaFieldFinder;
/**
@@ -39,7 +39,7 @@ public class ConfigurationBasedPartitionSpecExtractor implements HudiSourceParti
private final HudiSourceConfigImpl config;
@Override
- public List<OnePartitionField> spec(OneSchema tableSchema) {
+ public List<OnePartitionField> spec(InternalSchema tableSchema) {
List<OnePartitionField> partitionFields =
new ArrayList<>(config.getPartitionFieldSpecs().size());
for (HudiSourceConfigImpl.PartitionFieldSpec fieldSpec : config.getPartitionFieldSpecs()) {
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
index 1d6c1c7d..9b3da689 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiFileStatsExtractor.java
@@ -48,8 +48,8 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.MetadataPartitionType;
import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.Range;
@@ -84,7 +84,7 @@ public class HudiFileStatsExtractor {
* @return a stream of files with column stats and row count information
*/
public Stream<InternalDataFile> addStatsToFiles(
- HoodieTableMetadata metadataTable, Stream<InternalDataFile> files, OneSchema schema) {
+ HoodieTableMetadata metadataTable, Stream<InternalDataFile> files, InternalSchema schema) {
boolean useMetadataTableColStats =
metadataTable != null
&& metaClient
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
index b2c2bb47..d6d0f0e2 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
@@ -24,7 +24,7 @@ import java.util.Map;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.xtable.model.OneTable;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.spi.extractor.SchemaCatalogExtractor;
@@ -39,7 +39,7 @@ public class HudiSchemaCatalogExtractor implements SchemaCatalogExtractor<Hoodie
public static SchemaCatalog catalogWithTableSchema(OneTable table) {
// does not support schema versions for now
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
schemas.put(schemaVersion, table.getReadSchema());
return SchemaCatalog.builder().schemas(schemas).build();
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaExtractor.java
index 980a8271..7443d30f 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaExtractor.java
@@ -23,13 +23,13 @@ import org.apache.avro.Schema;
import com.google.common.annotations.VisibleForTesting;
import org.apache.xtable.avro.AvroSchemaConverter;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.spi.extractor.SchemaExtractor;
/**
- * Extracts the canonical {@link OneSchema} from {@link Schema} represented in Hudi. This uses logic
- * similar to how spark converts {@link Schema} to {@link
+ * Extracts the canonical {@link InternalSchema} from {@link Schema} represented in Hudi. This uses
+ * logic similar to how spark converts {@link Schema} to {@link
* org.apache.spark.sql.avro.SchemaConverters.SchemaType} in {@link
* org.apache.spark.sql.avro.SchemaConverters}
*
@@ -51,8 +51,8 @@ public class HudiSchemaExtractor implements SchemaExtractor<Schema> {
}
@Override
- public OneSchema schema(Schema schema) {
- return converter.toOneSchema(schema);
+ public InternalSchema schema(Schema schema) {
+ return converter.toInternalSchema(schema);
}
static String convertFromOneTablePath(String path) {
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
index aa89da93..20698171 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
@@ -34,9 +34,9 @@ import org.apache.hudi.common.util.Option;
import org.apache.xtable.exception.SchemaExtractorException;
import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.schema.SchemaFieldFinder;
@@ -57,7 +57,7 @@ public class HudiTableExtractor {
public OneTable table(HoodieTableMetaClient metaClient, HoodieInstant commit) {
TableSchemaResolver tableSchemaResolver = new TableSchemaResolver(metaClient);
- OneSchema canonicalSchema;
+ InternalSchema canonicalSchema;
Schema avroSchema;
try {
avroSchema = tableSchemaResolver.getTableAvroSchema(commit.getTimestamp());
@@ -89,7 +89,7 @@ public class HudiTableExtractor {
}
private List<OneField> getRecordKeyFields(
- HoodieTableMetaClient metaClient, OneSchema canonicalSchema) {
+ HoodieTableMetaClient metaClient, InternalSchema canonicalSchema) {
Option<String[]> recordKeyFieldNames = metaClient.getTableConfig().getRecordKeyFields();
if (!recordKeyFieldNames.isPresent()) {
return Collections.emptyList();
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java b/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
index 816d05ae..43032823 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
@@ -84,9 +84,9 @@ import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.OneIOException;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.OneFileGroup;
import org.apache.xtable.model.storage.TableFormat;
@@ -186,14 +186,14 @@ public class HudiTargetClient implements TargetClient {
}
@Override
- public void syncSchema(OneSchema schema) {
+ public void syncSchema(InternalSchema schema) {
if (metaClient.isPresent()) {
validateRecordKeysAreNotModified(schema);
}
- commitState.setSchema(avroSchemaConverter.fromOneSchema(schema));
+ commitState.setSchema(avroSchemaConverter.fromInternalSchema(schema));
}
- private void validateRecordKeysAreNotModified(OneSchema schema) {
+ private void validateRecordKeysAreNotModified(InternalSchema schema) {
Option<String[]> recordKeyFields = getMetaClient().getTableConfig().getRecordKeyFields();
if (recordKeyFields.isPresent()) {
List<String> existingHudiRecordKeys = Arrays.asList(recordKeyFields.get());
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
index 6e3eb2eb..df8051a8 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
@@ -46,8 +46,8 @@ import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.xtable.client.PerTableConfig;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.OneFileGroup;
import org.apache.xtable.model.storage.TableFormat;
@@ -164,7 +164,7 @@ public class IcebergClient implements TargetClient {
}
@Override
- public void syncSchema(OneSchema schema) {
+ public void syncSchema(InternalSchema schema) {
Schema latestSchema = schemaExtractor.toIceberg(schema);
schemaSync.sync(transaction.table().schema(), latestSchema, transaction);
}
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
index c6d0d3ad..18803e57 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
@@ -26,7 +26,7 @@ import lombok.Builder;
import org.apache.iceberg.DataFile;
import org.apache.xtable.exception.NotSupportedException;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.PartitionValue;
import org.apache.xtable.model.storage.FileFormat;
@@ -49,14 +49,14 @@ public class IcebergDataFileExtractor {
* @return corresponding OneTable data file
*/
InternalDataFile fromIceberg(
- DataFile dataFile, List<PartitionValue> partitionValues, OneSchema schema) {
+ DataFile dataFile, List<PartitionValue> partitionValues, InternalSchema schema) {
return fromIceberg(dataFile, partitionValues, schema, true);
}
private InternalDataFile fromIceberg(
DataFile dataFile,
List<PartitionValue> partitionValues,
- OneSchema schema,
+ InternalSchema schema,
boolean includeColumnStats) {
List<ColumnStat> columnStats =
includeColumnStats
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionSpecExtractor.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionSpecExtractor.java
index b98ef84d..4a4e811a 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionSpecExtractor.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionSpecExtractor.java
@@ -32,9 +32,9 @@ import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.types.Types;
import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.schema.SchemaFieldFinder;
@@ -114,7 +114,7 @@ public class IcebergPartitionSpecExtractor {
* @return generated internal representation of the Iceberg partition spec
*/
public List<OnePartitionField> fromIceberg(
- PartitionSpec iceSpec, Schema iceSchema, OneSchema irSchema) {
+ PartitionSpec iceSpec, Schema iceSchema, InternalSchema irSchema) {
if (iceSpec.isUnpartitioned()) {
return Collections.emptyList();
}
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergSchemaExtractor.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergSchemaExtractor.java
index 7d9b4861..9522672a 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergSchemaExtractor.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergSchemaExtractor.java
@@ -42,13 +42,13 @@ import org.apache.iceberg.types.Types;
import org.apache.xtable.collectors.CustomCollectors;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.SchemaExtractorException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
/**
* Schema extractor for Iceberg which converts canonical representation of the schema{@link
- * OneSchema} to Iceberg's schema representation {@link Schema}
+ * InternalSchema} to Iceberg's schema representation {@link Schema}
*/
@Log4j2
@NoArgsConstructor(access = AccessLevel.PRIVATE)
@@ -62,11 +62,11 @@ public class IcebergSchemaExtractor {
return INSTANCE;
}
- public Schema toIceberg(OneSchema oneSchema) {
+ public Schema toIceberg(InternalSchema internalSchema) {
// if field IDs are not assigned in the source, just use an incrementing integer
AtomicInteger fieldIdTracker = new AtomicInteger(0);
- List<Types.NestedField> nestedFields = convertFields(oneSchema, fieldIdTracker);
- List<OneField> recordKeyFields = oneSchema.getRecordKeyFields();
+ List<Types.NestedField> nestedFields = convertFields(internalSchema, fieldIdTracker);
+ List<OneField> recordKeyFields = internalSchema.getRecordKeyFields();
boolean recordKeyFieldsAreNotRequired =
recordKeyFields.stream().anyMatch(f -> f.getSchema().isNullable());
// Iceberg requires the identifier fields to be required fields, so if any of the record key
@@ -105,8 +105,8 @@ public class IcebergSchemaExtractor {
* @param iceSchema Iceberg schema
* @return Internal representation of deserialized iceberg schema
*/
- public OneSchema fromIceberg(Schema iceSchema) {
- return OneSchema.builder()
+ public InternalSchema fromIceberg(Schema iceSchema) {
+ return InternalSchema.builder()
.dataType(OneType.RECORD)
.fields(fromIceberg(iceSchema.columns(), null))
.name("record")
@@ -121,7 +121,7 @@ public class IcebergSchemaExtractor {
String doc = iceField.doc();
boolean isOptional = iceField.isOptional();
String fieldPath = getFullyQualifiedPath(parentPath, iceField.name());
- OneSchema irFieldSchema = fromIcebergType(type, fieldPath, doc, isOptional);
+ InternalSchema irFieldSchema = fromIcebergType(type, fieldPath, doc, isOptional);
return OneField.builder()
.name(iceField.name())
.fieldId(iceField.fieldId())
@@ -140,7 +140,8 @@ public class IcebergSchemaExtractor {
.replace(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME, LIST_ELEMENT_FIELD_NAME);
}
- private List<Types.NestedField> convertFields(OneSchema schema, AtomicInteger fieldIdTracker) {
+ private List<Types.NestedField> convertFields(
+ InternalSchema schema, AtomicInteger fieldIdTracker) {
// mirror iceberg pattern of assigning IDs for a level before recursing
List<Integer> ids =
schema.getFields().stream()
@@ -178,7 +179,7 @@ public class IcebergSchemaExtractor {
return Types.BinaryType.get();
case FIXED:
int size =
- (int) field.getSchema().getMetadata().get(OneSchema.MetadataKey.FIXED_BYTES_SIZE);
+ (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.FIXED_BYTES_SIZE);
return Types.FixedType.ofLength(size);
case BOOLEAN:
return Types.BooleanType.get();
@@ -192,8 +193,9 @@ public class IcebergSchemaExtractor {
return Types.DoubleType.get();
case DECIMAL:
int precision =
- (int) field.getSchema().getMetadata().get(OneSchema.MetadataKey.DECIMAL_PRECISION);
- int scale = (int) field.getSchema().getMetadata().get(OneSchema.MetadataKey.DECIMAL_SCALE);
+ (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_PRECISION);
+ int scale =
+ (int) field.getSchema().getMetadata().get(InternalSchema.MetadataKey.DECIMAL_SCALE);
return Types.DecimalType.of(precision, scale);
case RECORD:
return Types.StructType.of(convertFields(field.getSchema(), fieldIdTracker));
@@ -246,11 +248,11 @@ public class IcebergSchemaExtractor {
}
}
- private OneSchema fromIcebergType(
+ private InternalSchema fromIcebergType(
Type iceType, String fieldPath, String doc, boolean isOptional) {
OneType type;
List<OneField> fields = null;
- Map<OneSchema.MetadataKey, Object> metadata = null;
+ Map<InternalSchema.MetadataKey, Object> metadata = null;
switch (iceType.typeId()) {
case STRING:
type = OneType.STRING;
@@ -278,7 +280,8 @@ public class IcebergSchemaExtractor {
type = timestampType.shouldAdjustToUTC() ? OneType.TIMESTAMP : OneType.TIMESTAMP_NTZ;
metadata =
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS);
break;
case DOUBLE:
type = OneType.DOUBLE;
@@ -286,19 +289,20 @@ public class IcebergSchemaExtractor {
case DECIMAL:
Types.DecimalType decimalType = (Types.DecimalType) iceType;
metadata = new HashMap<>(2, 1.0f);
- metadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, decimalType.precision());
- metadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, decimalType.scale());
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, decimalType.precision());
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, decimalType.scale());
type = OneType.DECIMAL;
break;
case FIXED:
type = OneType.FIXED;
Types.FixedType fixedType = (Types.FixedType) iceType;
metadata =
- Collections.singletonMap(OneSchema.MetadataKey.FIXED_BYTES_SIZE, fixedType.length());
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.FIXED_BYTES_SIZE, fixedType.length());
break;
case UUID:
type = OneType.FIXED;
- metadata = Collections.singletonMap(OneSchema.MetadataKey.FIXED_BYTES_SIZE, 16);
+ metadata = Collections.singletonMap(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, 16);
break;
case STRUCT:
Types.StructType structType = (Types.StructType) iceType;
@@ -307,7 +311,7 @@ public class IcebergSchemaExtractor {
break;
case MAP:
Types.MapType mapType = (Types.MapType) iceType;
- OneSchema keySchema =
+ InternalSchema keySchema =
fromIcebergType(
mapType.keyType(),
getFullyQualifiedPath(fieldPath, OneField.Constants.MAP_KEY_FIELD_NAME),
@@ -320,7 +324,7 @@ public class IcebergSchemaExtractor {
.schema(keySchema)
.fieldId(mapType.keyId())
.build();
- OneSchema valueSchema =
+ InternalSchema valueSchema =
fromIcebergType(
mapType.valueType(),
getFullyQualifiedPath(fieldPath, OneField.Constants.MAP_VALUE_FIELD_NAME),
@@ -338,7 +342,7 @@ public class IcebergSchemaExtractor {
break;
case LIST:
Types.ListType listType = (Types.ListType) iceType;
- OneSchema elementSchema =
+ InternalSchema elementSchema =
fromIcebergType(
listType.elementType(),
getFullyQualifiedPath(fieldPath, OneField.Constants.ARRAY_ELEMENT_FIELD_NAME),
@@ -357,7 +361,7 @@ public class IcebergSchemaExtractor {
default:
throw new NotSupportedException("Unsupported type: " + iceType.typeId());
}
- return OneSchema.builder()
+ return InternalSchema.builder()
.name(iceType.typeId().name().toLowerCase())
.dataType(type)
.comment(doc)
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
index 5ff4a8ed..a0c0f37d 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
@@ -43,8 +43,8 @@ import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.model.stat.PartitionValue;
@@ -95,7 +95,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
Table iceTable = getSourceTable();
Schema iceSchema = iceTable.schemas().get(snapshot.schemaId());
IcebergSchemaExtractor schemaExtractor = IcebergSchemaExtractor.getInstance();
- OneSchema irSchema = schemaExtractor.fromIceberg(iceSchema);
+ InternalSchema irSchema = schemaExtractor.fromIceberg(iceSchema);
// TODO select snapshot specific partition spec
IcebergPartitionSpecExtractor partitionExtractor = IcebergPartitionSpecExtractor.getInstance();
@@ -124,8 +124,8 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
Integer iceSchemaId = snapshot.schemaId();
Schema iceSchema = iceTable.schemas().get(iceSchemaId);
IcebergSchemaExtractor schemaExtractor = IcebergSchemaExtractor.getInstance();
- OneSchema irSchema = schemaExtractor.fromIceberg(iceSchema);
- Map<SchemaVersion, OneSchema> catalog =
+ InternalSchema irSchema = schemaExtractor.fromIceberg(iceSchema);
+ Map<SchemaVersion, InternalSchema> catalog =
Collections.singletonMap(new SchemaVersion(iceSchemaId, ""), irSchema);
return SchemaCatalog.builder().schemas(catalog).build();
}
diff --git a/core/src/main/java/org/apache/xtable/schema/SchemaFieldFinder.java b/core/src/main/java/org/apache/xtable/schema/SchemaFieldFinder.java
index 16001bdb..c0513218 100644
--- a/core/src/main/java/org/apache/xtable/schema/SchemaFieldFinder.java
+++ b/core/src/main/java/org/apache/xtable/schema/SchemaFieldFinder.java
@@ -21,12 +21,12 @@ package org.apache.xtable.schema;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
/**
- * SchemaFieldFinder finds the {@link OneField} in the given {@link OneSchema} identified by the
- * fully qualified path.
+ * SchemaFieldFinder finds the {@link OneField} in the given {@link InternalSchema} identified by
+ * the fully qualified path.
*/
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class SchemaFieldFinder {
@@ -44,11 +44,11 @@ public class SchemaFieldFinder {
* @param path dot separated path
* @return the field if it exists, otherwise returns null
*/
- public OneField findFieldByPath(OneSchema schema, String path) {
+ public OneField findFieldByPath(InternalSchema schema, String path) {
return findFieldByPath(schema, path.split("\\."), 0);
}
- private OneField findFieldByPath(OneSchema schema, String[] pathParts, int startIndex) {
+ private OneField findFieldByPath(InternalSchema schema, String[] pathParts, int startIndex) {
if (pathParts.length == 0) {
return null;
}
diff --git a/core/src/test/java/org/apache/xtable/avro/TestAvroSchemaConverter.java b/core/src/test/java/org/apache/xtable/avro/TestAvroSchemaConverter.java
index 761622b8..e1b172a4 100644
--- a/core/src/test/java/org/apache/xtable/avro/TestAvroSchemaConverter.java
+++ b/core/src/test/java/org/apache/xtable/avro/TestAvroSchemaConverter.java
@@ -31,8 +31,8 @@ import org.junit.jupiter.api.Test;
import org.apache.hudi.common.util.Option;
import org.apache.xtable.hudi.idtracking.IdTracker;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
public class TestAvroSchemaConverter {
@@ -45,13 +45,14 @@ public class TestAvroSchemaConverter {
.parse(
"{\"type\":\"record\",\"name\":\"testRecord\",\"doc\":\"What's up doc\",\"fields\":[{\"name\":\"requiredBoolean\",\"type\":\"boolean\",\"default\":false},{\"name\":\"optionalBoolean\",\"type\":[\"null\",\"boolean\"],\"default\":null},{\"name\":\"requiredInt\",\"type\":\"int\",\"default\":123},{\"name\":\"optionalInt\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"requiredLong\",\"type\":\"long\"},{\"name\":\"optionalLong\",\"type\":[\"null\",\"long\"],\"defaul [...]
- Map<OneSchema.MetadataKey, Object> requiredEnumMetadata = new HashMap<>();
- requiredEnumMetadata.put(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
- Map<OneSchema.MetadataKey, Object> optionalEnumMetadata = new HashMap<>();
- optionalEnumMetadata.put(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
+ Map<InternalSchema.MetadataKey, Object> requiredEnumMetadata = new HashMap<>();
+ requiredEnumMetadata.put(InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
+ Map<InternalSchema.MetadataKey, Object> optionalEnumMetadata = new HashMap<>();
+ optionalEnumMetadata.put(
+ InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name(schemaName)
.comment(doc)
.dataType(OneType.RECORD)
@@ -61,7 +62,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(false)
@@ -71,7 +72,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(true)
@@ -81,7 +82,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -91,7 +92,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(true)
@@ -101,7 +102,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(false)
@@ -110,7 +111,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(true)
@@ -120,7 +121,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -129,7 +130,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(true)
@@ -139,7 +140,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(false)
@@ -148,7 +149,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(true)
@@ -158,7 +159,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -167,7 +168,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -177,7 +178,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("bytes")
.dataType(OneType.BYTES)
.isNullable(false)
@@ -186,7 +187,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("bytes")
.dataType(OneType.BYTES)
.isNullable(true)
@@ -196,7 +197,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("requiredEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("REQUIRED_ENUM")
.dataType(OneType.ENUM)
.isNullable(false)
@@ -207,7 +208,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("optionalEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("OPTIONAL_ENUM")
.dataType(OneType.ENUM)
.isNullable(true)
@@ -218,10 +219,9 @@ public class TestAvroSchemaConverter {
.build();
assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(avroRepresentation));
+ internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(avroRepresentation));
assertEquals(
- avroRepresentation,
- AvroSchemaConverter.getInstance().fromOneSchema(oneSchemaRepresentation));
+ avroRepresentation, AvroSchemaConverter.getInstance().fromInternalSchema(internalSchema));
}
@Test
@@ -231,8 +231,8 @@ public class TestAvroSchemaConverter {
.parse(
"{\"type\":\"record\",\"name\":\"testRecord\",\"fields\":[{\"name\":\"nestedOne\",\"type\":[\"null\",{\"type\":\"record\",\"name\":\"nestedOneType\",\"namespace\":\"nestedOne\",\"fields\":[{\"name\":\"nestedOptionalInt\",\"type\":[\"null\",\"int\"],\"default\":null},{\"name\":\"nestedRequiredDouble\",\"type\":\"double\"},{\"name\":\"nestedTwo\",\"type\":{\"type\":\"record\",\"name\":\"nestedTwoType\",\"namespace\":\"nestedOne.nestedTwo\",\"fields\":[{\"name\":\"doublyNest [...]
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("testRecord")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -242,7 +242,7 @@ public class TestAvroSchemaConverter {
.name("nestedOne")
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("nestedOneType")
.dataType(OneType.RECORD)
.isNullable(true)
@@ -252,7 +252,7 @@ public class TestAvroSchemaConverter {
.name("nestedOptionalInt")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(true)
@@ -263,7 +263,7 @@ public class TestAvroSchemaConverter {
.name("nestedRequiredDouble")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -273,7 +273,7 @@ public class TestAvroSchemaConverter {
.name("nestedTwo")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("nestedTwoType")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -283,7 +283,7 @@ public class TestAvroSchemaConverter {
.name("doublyNestedString")
.parentPath("nestedOne.nestedTwo")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -299,10 +299,9 @@ public class TestAvroSchemaConverter {
.build();
assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(avroRepresentation));
+ internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(avroRepresentation));
assertEquals(
- avroRepresentation,
- AvroSchemaConverter.getInstance().fromOneSchema(oneSchemaRepresentation));
+ avroRepresentation, AvroSchemaConverter.getInstance().fromInternalSchema(internalSchema));
}
@Test
@@ -312,8 +311,8 @@ public class TestAvroSchemaConverter {
.parse(
"{\"type\":\"record\",\"name\":\"testRecord\",\"fields\":[{\"name\":\"intList\",\"type\":{\"type\":\"array\",\"items\":\"int\"}},{\"name\":\"recordList\",\"type\":[\"null\",{\"type\":\"array\",\"items\":{\"type\":\"record\",\"name\":\"element\",\"namespace\":\"recordList._one_field_element\",\"fields\":[{\"name\":\"requiredDouble\",\"type\":\"double\"},{\"name\":\"optionalString\",\"type\":[\"null\",\"string\"],\"default\":null}]}}],\"default\":null}]}");
- OneSchema recordListElementSchema =
- OneSchema.builder()
+ InternalSchema recordListElementSchema =
+ InternalSchema.builder()
.name("element")
.isNullable(false)
.fields(
@@ -322,7 +321,7 @@ public class TestAvroSchemaConverter {
.name("requiredDouble")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -332,7 +331,7 @@ public class TestAvroSchemaConverter {
.name("optionalString")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -341,8 +340,8 @@ public class TestAvroSchemaConverter {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("testRecord")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -351,7 +350,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("intList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(false)
.dataType(OneType.LIST)
@@ -361,7 +360,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("intList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -372,7 +371,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("recordList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(true)
.dataType(OneType.LIST)
@@ -389,10 +388,9 @@ public class TestAvroSchemaConverter {
.build();
assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(avroRepresentation));
+ internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(avroRepresentation));
assertEquals(
- avroRepresentation,
- AvroSchemaConverter.getInstance().fromOneSchema(oneSchemaRepresentation));
+ avroRepresentation, AvroSchemaConverter.getInstance().fromInternalSchema(internalSchema));
}
@Test
@@ -402,8 +400,8 @@ public class TestAvroSchemaConverter {
.parse(
"{\"type\":\"record\",\"name\":\"testRecord\",\"fields\":[{\"name\":\"intMap\",\"type\":{\"type\":\"map\",\"values\":\"int\"}},{\"name\":\"recordMap\",\"type\":[\"null\",{\"type\":\"map\",\"values\":{\"type\":\"record\",\"name\":\"element\",\"namespace\":\"recordMap._one_field_value\",\"fields\":[{\"name\":\"requiredDouble\",\"type\":\"double\"},{\"name\":\"optionalString\",\"type\":[\"null\",\"string\"],\"default\":null}]}}],\"default\":null}]}");
- OneSchema recordMapElementSchema =
- OneSchema.builder()
+ InternalSchema recordMapElementSchema =
+ InternalSchema.builder()
.name("element")
.isNullable(false)
.fields(
@@ -412,7 +410,7 @@ public class TestAvroSchemaConverter {
.name("requiredDouble")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -422,7 +420,7 @@ public class TestAvroSchemaConverter {
.name("optionalString")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -431,8 +429,8 @@ public class TestAvroSchemaConverter {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("testRecord")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -441,7 +439,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(false)
.dataType(OneType.MAP)
@@ -451,7 +449,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.STRING)
.isNullable(false)
@@ -462,7 +460,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -473,7 +471,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(true)
.dataType(OneType.MAP)
@@ -483,7 +481,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.STRING)
.isNullable(false)
@@ -501,10 +499,9 @@ public class TestAvroSchemaConverter {
.build();
assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(avroRepresentation));
+ internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(avroRepresentation));
assertEquals(
- avroRepresentation,
- AvroSchemaConverter.getInstance().fromOneSchema(oneSchemaRepresentation));
+ avroRepresentation, AvroSchemaConverter.getInstance().fromInternalSchema(internalSchema));
}
@Test
@@ -521,21 +518,23 @@ public class TestAvroSchemaConverter {
+ "{\"name\":\"fixed_decimal\",\"type\":{\"type\":\"fixed\",\"logicalType\":\"decimal\",\"name\":\"fixed_field\",\"size\":10,\"precision\":5,\"scale\":3}},"
+ "{\"name\":\"fixed_plain\",\"type\":{\"type\":\"fixed\",\"name\":\"fixed_plain_field\",\"size\":10}}]}");
- Map<OneSchema.MetadataKey, Object> bytesMetadata = new HashMap<>();
- bytesMetadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, 4);
- bytesMetadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, 2);
- Map<OneSchema.MetadataKey, Object> fixedDecimalMetadata = new HashMap<>();
- fixedDecimalMetadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, 5);
- fixedDecimalMetadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, 3);
- fixedDecimalMetadata.put(OneSchema.MetadataKey.FIXED_BYTES_SIZE, 10);
- Map<OneSchema.MetadataKey, Object> fixedMetadata = new HashMap<>();
- fixedMetadata.put(OneSchema.MetadataKey.FIXED_BYTES_SIZE, 10);
- Map<OneSchema.MetadataKey, Object> millisMetadata = new HashMap<>();
- millisMetadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
- Map<OneSchema.MetadataKey, Object> microsMetadata = new HashMap<>();
- microsMetadata.put(OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ Map<InternalSchema.MetadataKey, Object> bytesMetadata = new HashMap<>();
+ bytesMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 4);
+ bytesMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 2);
+ Map<InternalSchema.MetadataKey, Object> fixedDecimalMetadata = new HashMap<>();
+ fixedDecimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 5);
+ fixedDecimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 3);
+ fixedDecimalMetadata.put(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, 10);
+ Map<InternalSchema.MetadataKey, Object> fixedMetadata = new HashMap<>();
+ fixedMetadata.put(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, 10);
+ Map<InternalSchema.MetadataKey, Object> millisMetadata = new HashMap<>();
+ millisMetadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
+ Map<InternalSchema.MetadataKey, Object> microsMetadata = new HashMap<>();
+ microsMetadata.put(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("logicalTypes")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -544,7 +543,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("int_date")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.DATE)
.isNullable(false)
@@ -553,7 +552,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("long_timestamp_millis")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.isNullable(false)
@@ -563,7 +562,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("long_timestamp_micros")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.isNullable(false)
@@ -573,7 +572,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("long_timestamp_millis_local")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(false)
@@ -583,7 +582,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("long_timestamp_micros_local")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(false)
@@ -593,7 +592,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("bytes_decimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("bytes")
.dataType(OneType.DECIMAL)
.isNullable(false)
@@ -603,7 +602,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("fixed_decimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed_field")
.dataType(OneType.DECIMAL)
.isNullable(false)
@@ -613,7 +612,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("fixed_plain")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed_plain_field")
.dataType(OneType.FIXED)
.isNullable(false)
@@ -623,10 +622,9 @@ public class TestAvroSchemaConverter {
.build();
assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(avroRepresentation));
+ internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(avroRepresentation));
assertEquals(
- avroRepresentation,
- AvroSchemaConverter.getInstance().fromOneSchema(oneSchemaRepresentation));
+ avroRepresentation, AvroSchemaConverter.getInstance().fromInternalSchema(internalSchema));
}
@Test
@@ -641,8 +639,8 @@ public class TestAvroSchemaConverter {
Schema schemaWithIds =
IdTracker.getInstance().addIdTracking(avroRepresentation, Option.empty(), false);
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("Sample")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -651,7 +649,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("key")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -661,7 +659,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("ts")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(false)
@@ -671,7 +669,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("nested_record")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("Nested")
.dataType(OneType.RECORD)
.isNullable(true)
@@ -680,7 +678,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("nested_int")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -696,7 +694,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("nullable_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(true)
.dataType(OneType.MAP)
@@ -706,7 +704,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("nullable_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.STRING)
.isNullable(false)
@@ -718,7 +716,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("nullable_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("Nested")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -727,7 +725,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("nested_int")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -747,7 +745,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("primitive_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(false)
.dataType(OneType.MAP)
@@ -757,7 +755,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("primitive_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.STRING)
.isNullable(false)
@@ -769,7 +767,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("primitive_map_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -782,7 +780,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("array_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(false)
.dataType(OneType.LIST)
@@ -792,7 +790,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("array_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("Nested")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -801,7 +799,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("nested_int")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(false)
@@ -821,7 +819,7 @@ public class TestAvroSchemaConverter {
OneField.builder()
.name("primitive_array_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(false)
.dataType(OneType.LIST)
@@ -831,7 +829,7 @@ public class TestAvroSchemaConverter {
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("primitive_array_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -842,7 +840,6 @@ public class TestAvroSchemaConverter {
.fieldId(7)
.build()))
.build();
- assertEquals(
- oneSchemaRepresentation, AvroSchemaConverter.getInstance().toOneSchema(schemaWithIds));
+ assertEquals(internalSchema, AvroSchemaConverter.getInstance().toInternalSchema(schemaWithIds));
}
}
diff --git a/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java b/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
index 03cf602b..98344db1 100644
--- a/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
+++ b/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
@@ -60,9 +60,9 @@ import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.schema.SchemaCatalog;
@@ -79,7 +79,11 @@ public class ITDeltaSourceClient {
OneField.builder()
.name("col1")
.schema(
- OneSchema.builder().name("integer").dataType(OneType.INT).isNullable(true).build())
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(OneType.INT)
+ .isNullable(true)
+ .build())
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.build();
private static final ColumnStat COL1_COLUMN_STAT =
@@ -95,7 +99,11 @@ public class ITDeltaSourceClient {
OneField.builder()
.name("col2")
.schema(
- OneSchema.builder().name("integer").dataType(OneType.INT).isNullable(true).build())
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(OneType.INT)
+ .isNullable(true)
+ .build())
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.build();
private static final ColumnStat COL2_COLUMN_STAT =
@@ -174,14 +182,14 @@ public class ITDeltaSourceClient {
snapshot.getTable(),
tableName,
TableFormat.DELTA,
- OneSchema.builder().name("struct").dataType(OneType.RECORD).fields(fields).build(),
+ InternalSchema.builder().name("struct").dataType(OneType.RECORD).fields(fields).build(),
DataLayoutStrategy.FLAT,
"file:" + basePath,
Collections.emptyList());
// Validate schema catalog
- SchemaCatalog oneSchemaCatalog = snapshot.getSchemaCatalog();
+ SchemaCatalog schemaCatalog = snapshot.getSchemaCatalog();
validateSchemaCatalog(
- oneSchemaCatalog,
+ schemaCatalog,
Collections.singletonMap(new SchemaVersion(1, ""), snapshot.getTable().getReadSchema()));
// Validate data files
List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
@@ -231,7 +239,7 @@ public class ITDeltaSourceClient {
OneField.builder()
.name("part_col")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -243,7 +251,7 @@ public class ITDeltaSourceClient {
snapshot.getTable(),
tableName,
TableFormat.DELTA,
- OneSchema.builder().name("struct").dataType(OneType.RECORD).fields(fields).build(),
+ InternalSchema.builder().name("struct").dataType(OneType.RECORD).fields(fields).build(),
DataLayoutStrategy.HIVE_STYLE_PARTITION,
"file:" + basePath,
Collections.singletonList(
@@ -252,9 +260,9 @@ public class ITDeltaSourceClient {
.transformType(PartitionTransformType.VALUE)
.build()));
// Validate schema catalog
- SchemaCatalog oneSchemaCatalog = snapshot.getSchemaCatalog();
+ SchemaCatalog schemaCatalog = snapshot.getSchemaCatalog();
validateSchemaCatalog(
- oneSchemaCatalog,
+ schemaCatalog,
Collections.singletonMap(new SchemaVersion(1, ""), snapshot.getTable().getReadSchema()));
// Validate data files
List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
@@ -651,7 +659,7 @@ public class ITDeltaSourceClient {
OneTable oneTable,
String tableName,
String tableFormat,
- OneSchema readSchema,
+ InternalSchema readSchema,
DataLayoutStrategy dataLayoutStrategy,
String basePath,
List<OnePartitionField> partitioningFields) {
@@ -664,8 +672,8 @@ public class ITDeltaSourceClient {
}
private void validateSchemaCatalog(
- SchemaCatalog oneSchemaCatalog, Map<SchemaVersion, OneSchema> schemas) {
- Assertions.assertEquals(schemas, oneSchemaCatalog.getSchemas());
+ SchemaCatalog schemaCatalog, Map<SchemaVersion, InternalSchema> schemas) {
+ Assertions.assertEquals(schemas, schemaCatalog.getSchemas());
}
private void validatePartitionDataFiles(
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
index bc4b9928..de40d7ef 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaPartitionExtractor.java
@@ -34,9 +34,9 @@ import org.junit.jupiter.api.Test;
import scala.collection.JavaConverters;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.PartitionValue;
@@ -99,13 +99,14 @@ public class TestDeltaPartitionExtractor {
Metadata.fromJson("{\"delta.generationExpression\": \"HOUR(birthDate)\"}")));
}
};
- private static final OneSchema TIMESTAMP_SCHEMA =
- OneSchema.builder()
+ private static final InternalSchema TIMESTAMP_SCHEMA =
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
.build();
private final DeltaPartitionExtractor deltaPartitionExtractor =
@@ -116,9 +117,9 @@ public class TestDeltaPartitionExtractor {
public void testUnpartitionedTable() {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, new StructType());
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, new StructType());
assertTrue(onePartitionFields.isEmpty());
}
@@ -127,19 +128,23 @@ public class TestDeltaPartitionExtractor {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
StructType partitionSchema = getSchemaWithFields(Arrays.asList("gender"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
.sourceField(
OneField.builder()
.name("gender")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(OneType.STRING)
+ .build())
.build())
.transformType(PartitionTransformType.VALUE)
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -148,7 +153,7 @@ public class TestDeltaPartitionExtractor {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateOfBirth"));
StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateOfBirth"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
@@ -157,7 +162,7 @@ public class TestDeltaPartitionExtractor {
.partitionFieldNames(Collections.singletonList("dateOfBirth"))
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -166,7 +171,7 @@ public class TestDeltaPartitionExtractor {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateFmt"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
@@ -175,7 +180,7 @@ public class TestDeltaPartitionExtractor {
.partitionFieldNames(Collections.singletonList("dateFmt"))
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -184,7 +189,7 @@ public class TestDeltaPartitionExtractor {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
@@ -193,7 +198,7 @@ public class TestDeltaPartitionExtractor {
.partitionFieldNames(Collections.singletonList("yearOfBirth"))
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -202,7 +207,7 @@ public class TestDeltaPartitionExtractor {
StructType tableSchema =
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth", "id"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
@@ -214,12 +219,13 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("id")
- .schema(OneSchema.builder().name("integer").dataType(OneType.INT).build())
+ .schema(
+ InternalSchema.builder().name("integer").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -239,7 +245,7 @@ public class TestDeltaPartitionExtractor {
StructType partitionSchema =
getSchemaWithFields(
Arrays.asList("yearOfBirth", "monthOfBirth", "dayOfBirth", "hourOfBirth"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
@@ -249,7 +255,7 @@ public class TestDeltaPartitionExtractor {
.transformType(PartitionTransformType.HOUR)
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -260,14 +266,15 @@ public class TestDeltaPartitionExtractor {
getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
StructType partitionSchema =
getSchemaWithFields(Arrays.asList("id", "dateFmt", "gender", "dateOfBirth"));
- OneSchema oneSchema = deltaSchemaExtractor.toOneSchema(tableSchema);
+ InternalSchema internalSchema = deltaSchemaExtractor.toInternalSchema(tableSchema);
List<OnePartitionField> expectedOnePartitionFields =
Arrays.asList(
OnePartitionField.builder()
.sourceField(
OneField.builder()
.name("id")
- .schema(OneSchema.builder().name("integer").dataType(OneType.INT).build())
+ .schema(
+ InternalSchema.builder().name("integer").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build(),
@@ -280,7 +287,11 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("gender")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(OneType.STRING)
+ .build())
.build())
.transformType(PartitionTransformType.VALUE)
.build(),
@@ -290,7 +301,7 @@ public class TestDeltaPartitionExtractor {
.partitionFieldNames(Collections.singletonList("dateOfBirth"))
.build());
List<OnePartitionField> onePartitionFields =
- deltaPartitionExtractor.convertFromDeltaPartitionFormat(oneSchema, partitionSchema);
+ deltaPartitionExtractor.convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
assertEquals(expectedOnePartitionFields, onePartitionFields);
}
@@ -310,7 +321,8 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("partition_column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -319,7 +331,8 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("partition_column2")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -360,7 +373,8 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("partition_column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -370,7 +384,10 @@ public class TestDeltaPartitionExtractor {
OneField.builder()
.name("some_date_column")
.schema(
- OneSchema.builder().name("timestamp").dataType(OneType.TIMESTAMP).build())
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(OneType.TIMESTAMP)
+ .build())
.build())
.partitionFieldNames(Collections.singletonList("date_partition_column"))
.transformType(PartitionTransformType.HOUR)
@@ -413,7 +430,8 @@ public class TestDeltaPartitionExtractor {
.sourceField(
OneField.builder()
.name("partition_column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -423,7 +441,10 @@ public class TestDeltaPartitionExtractor {
OneField.builder()
.name("some_date_column")
.schema(
- OneSchema.builder().name("timestamp").dataType(OneType.TIMESTAMP).build())
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(OneType.TIMESTAMP)
+ .build())
.build())
.partitionFieldNames(
Arrays.asList(
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
index 03809894..10a1dbcc 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaSchemaExtractor.java
@@ -29,20 +29,20 @@ import org.apache.spark.sql.types.StructType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
public class TestDeltaSchemaExtractor {
@Test
public void testPrimitiveTypes() {
- Map<OneSchema.MetadataKey, Object> decimalMetadata = new HashMap<>();
- decimalMetadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, 10);
- decimalMetadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, 2);
+ Map<InternalSchema.MetadataKey, Object> decimalMetadata = new HashMap<>();
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 10);
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 2);
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -51,7 +51,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(false)
@@ -60,7 +60,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(true)
@@ -70,7 +70,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -79,7 +79,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(true)
@@ -89,7 +89,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(false)
@@ -98,7 +98,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(true)
@@ -108,7 +108,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -117,7 +117,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(true)
@@ -127,7 +127,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(false)
@@ -136,7 +136,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(true)
@@ -146,7 +146,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -155,7 +155,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -165,7 +165,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(false)
@@ -174,7 +174,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(true)
@@ -184,7 +184,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredDate")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("date")
.dataType(OneType.DATE)
.isNullable(false)
@@ -193,7 +193,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalDate")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("date")
.dataType(OneType.DATE)
.isNullable(true)
@@ -203,7 +203,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredDecimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("decimal")
.dataType(OneType.DECIMAL)
.isNullable(false)
@@ -213,7 +213,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalDecimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("decimal")
.dataType(OneType.DECIMAL)
.isNullable(true)
@@ -246,16 +246,15 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentation));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchema));
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
public void testFixedBytes() {
- OneSchema oneSchemaRepresentationOriginal =
- OneSchema.builder()
+ InternalSchema internalSchemaOriginal =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -264,7 +263,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed")
.dataType(OneType.FIXED)
.isNullable(false)
@@ -273,7 +272,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed")
.dataType(OneType.FIXED)
.isNullable(true)
@@ -282,8 +281,8 @@ public class TestDeltaSchemaExtractor {
.build()))
.build();
- OneSchema oneSchemaRepresentationAfterRoundTrip =
- OneSchema.builder()
+ InternalSchema internalSchemaAfterRoundTrip =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -292,7 +291,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(false)
@@ -301,7 +300,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(true)
@@ -316,19 +315,19 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentationOriginal));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchemaOriginal));
Assertions.assertEquals(
- oneSchemaRepresentationAfterRoundTrip,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchemaAfterRoundTrip,
+ DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
public void testTimestamps() {
- Map<OneSchema.MetadataKey, Object> metadata =
+ Map<InternalSchema.MetadataKey, Object> metadata =
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
- OneSchema oneSchemaRepresentationTimestamp =
- OneSchema.builder()
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
+ InternalSchema internalSchemaTimestamp =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -337,7 +336,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredTimestamp")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.isNullable(false)
@@ -347,7 +346,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalTimestamp")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.isNullable(true)
@@ -357,8 +356,8 @@ public class TestDeltaSchemaExtractor {
.build()))
.build();
- OneSchema oneSchemaRepresentationTimestampNtz =
- OneSchema.builder()
+ InternalSchema internalSchemaTimestampNtz =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -367,7 +366,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredTimestampNtz")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(false)
@@ -376,7 +375,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalTimestampNtz")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(true)
@@ -397,24 +396,25 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentationTimestamp,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentationTimestamp));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchemaTimestamp));
Assertions.assertEquals(
- oneSchemaRepresentationTimestamp,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentationTimestamp));
+ internalSchemaTimestamp,
+ DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentationTimestamp));
Assertions.assertEquals(
structRepresentationTimestampNtz,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentationTimestampNtz));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchemaTimestampNtz));
}
@Test
public void testEnums() {
- Map<OneSchema.MetadataKey, Object> requiredEnumMetadata = new HashMap<>();
- requiredEnumMetadata.put(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
- Map<OneSchema.MetadataKey, Object> optionalEnumMetadata = new HashMap<>();
- optionalEnumMetadata.put(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
+ Map<InternalSchema.MetadataKey, Object> requiredEnumMetadata = new HashMap<>();
+ requiredEnumMetadata.put(InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
+ Map<InternalSchema.MetadataKey, Object> optionalEnumMetadata = new HashMap<>();
+ optionalEnumMetadata.put(
+ InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -423,7 +423,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("requiredEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("REQUIRED_ENUM")
.dataType(OneType.ENUM)
.isNullable(false)
@@ -433,7 +433,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("optionalEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("OPTIONAL_ENUM")
.dataType(OneType.ENUM)
.isNullable(true)
@@ -450,13 +450,13 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentation));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchema));
}
@Test
public void testMaps() {
- OneSchema recordMapElementSchema =
- OneSchema.builder()
+ InternalSchema recordMapElementSchema =
+ InternalSchema.builder()
.name("struct")
.isNullable(true)
.fields(
@@ -465,7 +465,7 @@ public class TestDeltaSchemaExtractor {
.name("requiredDouble")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -475,7 +475,7 @@ public class TestDeltaSchemaExtractor {
.name("optionalString")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -484,8 +484,8 @@ public class TestDeltaSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -494,7 +494,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(false)
.dataType(OneType.MAP)
@@ -504,7 +504,7 @@ public class TestDeltaSchemaExtractor {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -514,7 +514,7 @@ public class TestDeltaSchemaExtractor {
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("intMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -525,7 +525,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(true)
.dataType(OneType.MAP)
@@ -535,7 +535,7 @@ public class TestDeltaSchemaExtractor {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -565,16 +565,15 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentation));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchema));
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
public void testLists() {
- OneSchema recordListElementSchema =
- OneSchema.builder()
+ InternalSchema recordListElementSchema =
+ InternalSchema.builder()
.name("struct")
.isNullable(true)
.fields(
@@ -583,7 +582,7 @@ public class TestDeltaSchemaExtractor {
.name("requiredDouble")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -593,7 +592,7 @@ public class TestDeltaSchemaExtractor {
.name("optionalString")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -602,8 +601,8 @@ public class TestDeltaSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -612,7 +611,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("intList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(false)
.dataType(OneType.LIST)
@@ -622,7 +621,7 @@ public class TestDeltaSchemaExtractor {
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("intList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -633,7 +632,7 @@ public class TestDeltaSchemaExtractor {
OneField.builder()
.name("recordList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(true)
.dataType(OneType.LIST)
@@ -659,16 +658,15 @@ public class TestDeltaSchemaExtractor {
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentation));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchema));
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
public void testNestedRecords() {
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -678,7 +676,7 @@ public class TestDeltaSchemaExtractor {
.name("nestedOne")
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(true)
@@ -688,7 +686,7 @@ public class TestDeltaSchemaExtractor {
.name("nestedOptionalInt")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(true)
@@ -699,7 +697,7 @@ public class TestDeltaSchemaExtractor {
.name("nestedRequiredDouble")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -709,7 +707,7 @@ public class TestDeltaSchemaExtractor {
.name("nestedTwo")
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -719,7 +717,7 @@ public class TestDeltaSchemaExtractor {
.name("doublyNestedString")
.parentPath("nestedOne.nestedTwo")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -748,10 +746,9 @@ public class TestDeltaSchemaExtractor {
true);
Assertions.assertEquals(
structRepresentation,
- DeltaSchemaExtractor.getInstance().fromOneSchema(oneSchemaRepresentation));
+ DeltaSchemaExtractor.getInstance().fromInternalSchema(internalSchema));
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
@@ -784,8 +781,8 @@ public class TestDeltaSchemaExtractor {
true,
Metadata.fromJson("{\"delta.columnMapping.id\": 2}"));
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -796,7 +793,7 @@ public class TestDeltaSchemaExtractor {
.fieldId(2)
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(true)
@@ -807,7 +804,7 @@ public class TestDeltaSchemaExtractor {
.fieldId(3)
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(true)
@@ -819,7 +816,7 @@ public class TestDeltaSchemaExtractor {
.fieldId(5)
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -830,7 +827,7 @@ public class TestDeltaSchemaExtractor {
.fieldId(10)
.parentPath("nestedOne")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -841,7 +838,7 @@ public class TestDeltaSchemaExtractor {
.fieldId(12)
.parentPath("nestedOne.nestedTwo")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -856,8 +853,7 @@ public class TestDeltaSchemaExtractor {
.build()))
.build();
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
@Test
@@ -870,27 +866,26 @@ public class TestDeltaSchemaExtractor {
DataTypes.TimestampType,
true,
Metadata.fromJson("{\"delta.generationExpression\":\"YEAR(birthDate)\"}"));
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("struct")
.fields(
Collections.singletonList(
OneField.builder()
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION,
- OneSchema.MetadataValue.MICROS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
.build())
.name("birthDate")
.build()))
.build();
Assertions.assertEquals(
- oneSchemaRepresentation,
- DeltaSchemaExtractor.getInstance().toOneSchema(structRepresentation));
+ internalSchema, DeltaSchemaExtractor.getInstance().toInternalSchema(structRepresentation));
}
}
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaStatsExtractor.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaStatsExtractor.java
index 3c38620a..afd34ad5 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaStatsExtractor.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaStatsExtractor.java
@@ -36,8 +36,8 @@ import org.apache.spark.sql.delta.actions.AddFile;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.Range;
@@ -49,7 +49,7 @@ public class TestDeltaStatsExtractor {
@Test
public void testDeltaStats() throws JsonProcessingException {
- OneSchema schema = ColumnStatMapUtil.getSchema();
+ InternalSchema schema = ColumnStatMapUtil.getSchema();
List<ColumnStat> columnStats = getColumnStats();
@@ -114,7 +114,7 @@ public class TestDeltaStatsExtractor {
@Test
void roundTripStatsConversion() throws IOException {
- OneSchema schema = ColumnStatMapUtil.getSchema();
+ InternalSchema schema = ColumnStatMapUtil.getSchema();
List<OneField> fields = schema.getAllFields();
List<ColumnStat> columnStats = getColumnStats();
@@ -185,30 +185,30 @@ public class TestDeltaStatsExtractor {
return Arrays.asList(
OneField.builder()
.name("top_level_string")
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build(),
OneField.builder()
.name("nested")
- .schema(OneSchema.builder().dataType(OneType.RECORD).build())
+ .schema(InternalSchema.builder().dataType(OneType.RECORD).build())
.build(),
OneField.builder()
.name("int_field")
.parentPath("nested")
- .schema(OneSchema.builder().dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().dataType(OneType.INT).build())
.build(),
OneField.builder()
.name("double_nesting")
.parentPath("nested")
- .schema(OneSchema.builder().dataType(OneType.RECORD).build())
+ .schema(InternalSchema.builder().dataType(OneType.RECORD).build())
.build(),
OneField.builder()
.name("double_field")
.parentPath("nested.double_nesting")
- .schema(OneSchema.builder().dataType(OneType.DOUBLE).build())
+ .schema(InternalSchema.builder().dataType(OneType.DOUBLE).build())
.build(),
OneField.builder()
.name("top_level_int")
- .schema(OneSchema.builder().dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().dataType(OneType.INT).build())
.build());
}
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
index 5bd352cf..0149e8f8 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
@@ -79,9 +79,9 @@ import io.delta.standalone.types.StringType;
import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.PartitionValue;
@@ -137,15 +137,19 @@ public class TestDeltaSync {
@Test
public void testCreateSnapshotControlFlow() throws Exception {
- OneSchema schema1 = getOneSchema();
+ InternalSchema schema1 = getInternalSchema();
List<OneField> fields2 = new ArrayList<>(schema1.getFields());
fields2.add(
OneField.builder()
.name("float_field")
.schema(
- OneSchema.builder().name("float").dataType(OneType.FLOAT).isNullable(true).build())
+ InternalSchema.builder()
+ .name("float")
+ .dataType(OneType.FLOAT)
+ .isNullable(true)
+ .build())
.build());
- OneSchema schema2 = getOneSchema().toBuilder().fields(fields2).build();
+ InternalSchema schema2 = getInternalSchema().toBuilder().fields(fields2).build();
OneTable table1 = getOneTable(tableName, basePath, schema1, null, LAST_COMMIT_TIME);
OneTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
@@ -165,13 +169,14 @@ public class TestDeltaSync {
@Test
public void testPrimitiveFieldPartitioning() throws Exception {
- OneSchema schema = getOneSchema();
+ InternalSchema schema = getInternalSchema();
OnePartitionField onePartitionField =
OnePartitionField.builder()
.sourceField(
OneField.builder()
.name("string_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -210,13 +215,14 @@ public class TestDeltaSync {
@Test
public void testMultipleFieldPartitioning() throws Exception {
- OneSchema schema = getOneSchema();
+ InternalSchema schema = getInternalSchema();
OnePartitionField onePartitionField1 =
OnePartitionField.builder()
.sourceField(
OneField.builder()
.name("string_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -225,7 +231,7 @@ public class TestDeltaSync {
.sourceField(
OneField.builder()
.name("int_field")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -289,7 +295,7 @@ public class TestDeltaSync {
@ParameterizedTest
@MethodSource(value = "timestampPartitionTestingArgs")
public void testTimestampPartitioning(PartitionTransformType transformType) throws Exception {
- OneSchema schema = getOneSchema();
+ InternalSchema schema = getInternalSchema();
OnePartitionField partitionField =
OnePartitionField.builder()
.sourceField(SchemaFieldFinder.getInstance().findFieldByPath(schema, "timestamp_field"))
@@ -399,7 +405,7 @@ public class TestDeltaSync {
private OneTable getOneTable(
String tableName,
Path basePath,
- OneSchema schema,
+ InternalSchema schema,
List<OnePartitionField> partitionFields,
Instant lastCommitTime) {
return OneTable.builder()
@@ -428,11 +434,11 @@ public class TestDeltaSync {
.build();
}
- private OneSchema getOneSchema() {
- Map<OneSchema.MetadataKey, Object> timestampMetadata = new HashMap<>();
+ private InternalSchema getInternalSchema() {
+ Map<InternalSchema.MetadataKey, Object> timestampMetadata = new HashMap<>();
timestampMetadata.put(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
- return OneSchema.builder()
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
+ return InternalSchema.builder()
.dataType(OneType.RECORD)
.name("top_level_schema")
.fields(
@@ -440,7 +446,7 @@ public class TestDeltaSync {
OneField.builder()
.name("long_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(true)
@@ -449,7 +455,7 @@ public class TestDeltaSync {
OneField.builder()
.name("string_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -458,7 +464,7 @@ public class TestDeltaSync {
OneField.builder()
.name("int_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("int")
.dataType(OneType.INT)
.isNullable(true)
@@ -467,7 +473,7 @@ public class TestDeltaSync {
OneField.builder()
.name("timestamp_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("time")
.dataType(OneType.TIMESTAMP)
.isNullable(true)
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaValueConverter.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaValueConverter.java
index e8e96f8f..e633f0a4 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaValueConverter.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaValueConverter.java
@@ -33,7 +33,7 @@ import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
@@ -42,7 +42,7 @@ public class TestDeltaValueConverter {
@ParameterizedTest
@MethodSource("valuesWithSchemaProviderForColStats")
public void formattedValueDifferentTypesForColStats(
- Object fieldValue, OneSchema fieldSchema, Object expectedDeltaValue) {
+ Object fieldValue, InternalSchema fieldSchema, Object expectedDeltaValue) {
Object deltaRepresentation =
DeltaValueConverter.convertToDeltaColumnStatValue(fieldValue, fieldSchema);
Object internalRepresentation =
@@ -85,56 +85,63 @@ public class TestDeltaValueConverter {
private static Stream<Arguments> valuesWithSchemaProviderForColStats() {
return Stream.of(
Arguments.of(
- null, OneSchema.builder().name("string").dataType(OneType.STRING).build(), null),
+ null, InternalSchema.builder().name("string").dataType(OneType.STRING).build(), null),
Arguments.of(
"some value",
- OneSchema.builder().name("string").dataType(OneType.STRING).build(),
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build(),
"some value"),
- Arguments.of(23L, OneSchema.builder().name("long").dataType(OneType.LONG).build(), 23L),
Arguments.of(
- 25.5, OneSchema.builder().name("double").dataType(OneType.DOUBLE).build(), 25.5),
+ 23L, InternalSchema.builder().name("long").dataType(OneType.LONG).build(), 23L),
Arguments.of(
- 18181, OneSchema.builder().name("int").dataType(OneType.DATE).build(), "2019-10-12"),
+ 25.5, InternalSchema.builder().name("double").dataType(OneType.DOUBLE).build(), 25.5),
Arguments.of(
- true, OneSchema.builder().name("boolean").dataType(OneType.BOOLEAN).build(), true),
+ 18181,
+ InternalSchema.builder().name("int").dataType(OneType.DATE).build(),
+ "2019-10-12"),
+ Arguments.of(
+ true, InternalSchema.builder().name("boolean").dataType(OneType.BOOLEAN).build(), true),
Arguments.of(
1665263297000L,
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MILLIS))
.build(),
"2022-10-08 21:08:17"),
Arguments.of(
1665263297000000L,
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
.build(),
"2022-10-08 21:08:17"),
Arguments.of(
1665263297000L,
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MILLIS))
.build(),
"2022-10-08 21:08:17"),
Arguments.of(
1665263297000000L,
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
.build(),
"2022-10-08 21:08:17"));
}
diff --git a/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java b/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
index 8b4fc533..ddfde08f 100644
--- a/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
+++ b/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
@@ -73,9 +73,9 @@ import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.ColumnStat;
@@ -105,8 +105,8 @@ public class ITHudiTargetClient {
private static final long FILE_SIZE = 100L;
private static final long RECORD_COUNT = 200L;
private static final long LAST_MODIFIED = System.currentTimeMillis();
- private static final OneSchema STRING_SCHEMA =
- OneSchema.builder().name("string").dataType(OneType.STRING).isNullable(false).build();
+ private static final InternalSchema STRING_SCHEMA =
+ InternalSchema.builder().name("string").dataType(OneType.STRING).isNullable(false).build();
private static final OneField PARTITION_FIELD_SOURCE =
OneField.builder().name(PARTITION_FIELD_NAME).schema(STRING_SCHEMA).build();
@@ -117,8 +117,8 @@ public class ITHudiTargetClient {
.transformType(PartitionTransformType.VALUE)
.build();
private static final String TEST_SCHEMA_NAME = "test_schema";
- private static final OneSchema SCHEMA =
- OneSchema.builder()
+ private static final InternalSchema SCHEMA =
+ InternalSchema.builder()
.name(TEST_SCHEMA_NAME)
.dataType(OneType.RECORD)
.fields(
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java b/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
index 83efd1ba..1861cc8a 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
@@ -52,9 +52,9 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.CachingPath;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.ColumnStat;
@@ -79,7 +79,7 @@ public class TestBaseFileUpdatesExtractor {
.sourceField(
OneField.builder()
.name("string_field")
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java
index bb0d08a4..f42f6e2e 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiFileStatsExtractor.java
@@ -66,8 +66,8 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.xtable.GenericTable;
import org.apache.xtable.TestJavaHudiTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.model.stat.ColumnStat;
@@ -84,7 +84,7 @@ public class TestHudiFileStatsExtractor {
private final Configuration configuration = new Configuration();
private final OneField nestedIntBase = getNestedIntBase();
- private final OneSchema nestedSchema = getNestedSchema(nestedIntBase, "nested_record");
+ private final InternalSchema nestedSchema = getNestedSchema(nestedIntBase, "nested_record");
private final OneField longField = getLongField();
private final OneField stringField = getStringField();
private final OneField dateField = getDateField();
@@ -94,8 +94,8 @@ public class TestHudiFileStatsExtractor {
private final OneField arrayField = getArrayField(nestedIntBase);
private final OneField decimalField = getDecimalField();
- private final OneSchema schema =
- OneSchema.builder()
+ private final InternalSchema schema =
+ InternalSchema.builder()
.name("schema")
.fields(
Arrays.asList(
@@ -107,14 +107,16 @@ public class TestHudiFileStatsExtractor {
OneField.builder()
.name("map_record")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.fields(Arrays.asList(mapKeyField, mapValueField))
.build())
.build(),
OneField.builder()
.name("repeated_record")
.schema(
- OneSchema.builder().fields(Collections.singletonList(arrayField)).build())
+ InternalSchema.builder()
+ .fields(Collections.singletonList(arrayField))
+ .build())
.build(),
decimalField))
.build();
@@ -328,13 +330,13 @@ public class TestHudiFileStatsExtractor {
}
private OneField getDecimalField() {
- Map<OneSchema.MetadataKey, Object> metadata = new HashMap<>();
- metadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, 20);
- metadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, 2);
+ Map<InternalSchema.MetadataKey, Object> metadata = new HashMap<>();
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 20);
+ metadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 2);
return OneField.builder()
.name("decimal_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("decimal")
.dataType(OneType.DECIMAL)
.metadata(metadata)
@@ -365,47 +367,48 @@ public class TestHudiFileStatsExtractor {
return OneField.builder()
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("map_record")
- .schema(OneSchema.builder().name("map_key").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("map_key").dataType(OneType.STRING).build())
.build();
}
private OneField getTimestampField() {
return OneField.builder()
.name("timestamp_field")
- .schema(OneSchema.builder().name("time").dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().name("time").dataType(OneType.TIMESTAMP_NTZ).build())
.build();
}
private OneField getDateField() {
return OneField.builder()
.name("date_field")
- .schema(OneSchema.builder().name("date").dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().name("date").dataType(OneType.DATE).build())
.build();
}
private OneField getStringField() {
return OneField.builder()
.name("key")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
}
private OneField getLongField() {
return OneField.builder()
.name("long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build();
}
private OneField getNestedIntBase() {
return OneField.builder()
.name("nested_int")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).isNullable(false).build())
+ .schema(
+ InternalSchema.builder().name("int").dataType(OneType.INT).isNullable(false).build())
.build();
}
- private OneSchema getNestedSchema(OneField nestedIntBase, String parentPath) {
- return OneSchema.builder()
+ private InternalSchema getNestedSchema(OneField nestedIntBase, String parentPath) {
+ return InternalSchema.builder()
.name("nested")
.dataType(OneType.RECORD)
.fields(Collections.singletonList(nestedIntBase.toBuilder().parentPath(parentPath).build()))
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiPartitionValuesExtractor.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiPartitionValuesExtractor.java
index db0de953..3fc25e0e 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiPartitionValuesExtractor.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiPartitionValuesExtractor.java
@@ -33,9 +33,9 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.apache.xtable.exception.PartitionValuesExtractorException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.PartitionValue;
@@ -50,7 +50,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -72,7 +73,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -203,7 +205,10 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("col")
.schema(
- OneSchema.builder().name("partition-column").dataType(columnType).build())
+ InternalSchema.builder()
+ .name("partition-column")
+ .dataType(columnType)
+ .build())
.build())
.transformType(partitionTransformType)
.build();
@@ -225,15 +230,16 @@ public class TestHudiPartitionValuesExtractor {
@Test
public void testMultipleColumns() {
- Map<OneSchema.MetadataKey, Object> timeFieldMetadata = new HashMap<>();
+ Map<InternalSchema.MetadataKey, Object> timeFieldMetadata = new HashMap<>();
timeFieldMetadata.put(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
OnePartitionField column1 =
OnePartitionField.builder()
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -243,7 +249,7 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("column2")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("time")
.dataType(OneType.TIMESTAMP)
.metadata(timeFieldMetadata)
@@ -256,7 +262,7 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column3")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -285,7 +291,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -294,7 +301,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column2")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.DAY)
.build();
@@ -303,7 +311,7 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column3")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -330,7 +338,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -340,7 +349,7 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("column2")
.parentPath("base")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -363,7 +372,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -373,7 +383,7 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("column2")
.parentPath("base")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -397,7 +407,8 @@ public class TestHudiPartitionValuesExtractor {
.sourceField(
OneField.builder()
.name("column1")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(
+ InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -407,7 +418,7 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("column2")
.parentPath("base")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build();
@@ -444,7 +455,10 @@ public class TestHudiPartitionValuesExtractor {
OneField.builder()
.name("column1")
.schema(
- OneSchema.builder().name("time").dataType(OneType.TIMESTAMP_NTZ).build())
+ InternalSchema.builder()
+ .name("time")
+ .dataType(OneType.TIMESTAMP_NTZ)
+ .build())
.build())
.transformType(PartitionTransformType.DAY)
.build();
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiSchemaExtractor.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiSchemaExtractor.java
index 92ebb2ec..9140a431 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiSchemaExtractor.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiSchemaExtractor.java
@@ -28,18 +28,18 @@ import org.apache.avro.SchemaBuilder;
import org.junit.jupiter.api.Test;
import org.apache.xtable.avro.AvroSchemaConverter;
-import org.apache.xtable.model.schema.OneSchema;
+import org.apache.xtable.model.schema.InternalSchema;
public class TestHudiSchemaExtractor {
@Test
public void testHoodieTableSchema() {
AvroSchemaConverter mockConverter = mock(AvroSchemaConverter.class);
- OneSchema mockOutput = OneSchema.builder().name("fake schema").build();
+ InternalSchema mockOutput = InternalSchema.builder().name("fake schema").build();
Schema schema = SchemaBuilder.record("fake").fields().requiredString("foo").endRecord();
- when(mockConverter.toOneSchema(schema)).thenReturn(mockOutput);
- OneSchema canonicalSchema = new HudiSchemaExtractor(mockConverter).schema(schema);
+ when(mockConverter.toInternalSchema(schema)).thenReturn(mockOutput);
+ InternalSchema canonicalSchema = new HudiSchemaExtractor(mockConverter).schema(schema);
assertEquals(mockOutput, canonicalSchema);
- verify(mockConverter).toOneSchema(schema);
+ verify(mockConverter).toInternalSchema(schema);
}
}
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
index 19e20c88..eb720844 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
@@ -39,9 +39,9 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.TableFormat;
@@ -62,8 +62,8 @@ public class TestHudiTableManager {
String field1 = "field1";
String field2 = "field2";
String recordKeyField = "path1.path2";
- OneSchema tableSchema =
- OneSchema.builder()
+ InternalSchema tableSchema =
+ InternalSchema.builder()
.fields(
Arrays.asList(
OneField.builder().name(field1).build(),
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
index 00b28112..b0813770 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
@@ -44,9 +44,9 @@ import org.apache.xtable.avro.AvroSchemaConverter;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.DataFilesDiff;
@@ -94,15 +94,15 @@ public class TestHudiTargetClient {
void syncSchema() {
HudiTargetClient targetClient = getTargetClient(null);
HudiTargetClient.CommitState mockCommitState = initMocksForBeginSync(targetClient).getLeft();
- OneSchema input =
- OneSchema.builder()
+ InternalSchema input =
+ InternalSchema.builder()
.name("schema")
.dataType(OneType.RECORD)
.recordKeyFields(
Collections.singletonList(OneField.builder().name("record_key_field").build()))
.build();
Schema converted = SchemaBuilder.record("record").fields().requiredInt("field").endRecord();
- when(mockAvroSchemaConverter.fromOneSchema(input)).thenReturn(converted);
+ when(mockAvroSchemaConverter.fromInternalSchema(input)).thenReturn(converted);
targetClient.syncSchema(input);
// validate that schema is set in commitState
verify(mockCommitState).setSchema(converted);
@@ -116,14 +116,14 @@ public class TestHudiTargetClient {
Arrays.asList(
OneField.builder().name("record_key_field").build(),
OneField.builder().name("record_key_field2").build());
- OneSchema input =
- OneSchema.builder()
+ InternalSchema input =
+ InternalSchema.builder()
.name("schema")
.dataType(OneType.RECORD)
.recordKeyFields(recordKeyFields)
.build();
Schema converted = SchemaBuilder.record("record").fields().requiredInt("field").endRecord();
- when(mockAvroSchemaConverter.fromOneSchema(input)).thenReturn(converted);
+ when(mockAvroSchemaConverter.fromInternalSchema(input)).thenReturn(converted);
Exception thrownException =
assertThrows(NotSupportedException.class, () -> targetClient.syncSchema(input));
assertEquals(
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergColumnStatsConverter.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergColumnStatsConverter.java
index 72991d29..9e7d28e0 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergColumnStatsConverter.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergColumnStatsConverter.java
@@ -39,8 +39,8 @@ import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Conversions;
import org.apache.iceberg.types.Types;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.Range;
@@ -55,41 +55,41 @@ public class TestIcebergColumnStatsConverter {
OneField timestampField =
OneField.builder()
.name("timestamp_field")
- .schema(OneSchema.builder().name("time").dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().name("time").dataType(OneType.TIMESTAMP_NTZ).build())
.build();
OneField dateField =
OneField.builder()
.name("date_field")
- .schema(OneSchema.builder().name("date").dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().name("date").dataType(OneType.DATE).build())
.build();
OneField groupId =
OneField.builder()
.name("group_id")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build();
OneField stringField =
OneField.builder()
.name("string_field")
.parentPath("record")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
OneField mapFieldKey =
OneField.builder()
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("map_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
OneField mapFieldValue =
OneField.builder()
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("map_field")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build();
OneField arrayFieldElement =
OneField.builder()
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("array_field")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build();
ColumnStat timestampColumnStats =
ColumnStat.builder()
@@ -234,7 +234,7 @@ public class TestIcebergColumnStatsConverter {
OneField dateField =
OneField.builder()
.name("date_field")
- .schema(OneSchema.builder().name("date").dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().name("date").dataType(OneType.DATE).build())
.build();
ColumnStat dateColumnStats =
ColumnStat.builder()
@@ -310,32 +310,32 @@ public class TestIcebergColumnStatsConverter {
OneField.builder()
.fieldId(1)
.name("int_field")
- .schema(OneSchema.builder().dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().dataType(OneType.INT).build())
.build(),
OneField.builder()
.fieldId(2)
.name("not_tracked_field")
- .schema(OneSchema.builder().dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().dataType(OneType.DATE).build())
.build(),
OneField.builder()
.fieldId(3)
.name("null_field")
- .schema(OneSchema.builder().dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().dataType(OneType.INT).build())
.build(),
OneField.builder()
.fieldId(4)
.name("string_field")
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build(),
OneField.builder()
.fieldId(5)
.name("date_field")
- .schema(OneSchema.builder().dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().dataType(OneType.DATE).build())
.build(),
OneField.builder()
.fieldId(6)
.name("timestamp_field")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP).build())
.build());
List<ColumnStat> actual =
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionSpecExtractor.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionSpecExtractor.java
index d3420363..d1bf80db 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionSpecExtractor.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionSpecExtractor.java
@@ -31,9 +31,9 @@ import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
@@ -65,7 +65,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("timestamp_hour")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP).build())
.build())
.transformType(PartitionTransformType.HOUR)
.build(),
@@ -73,7 +73,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("string_field")
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build());
@@ -95,7 +95,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("timestamp_year")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
.build())
.transformType(PartitionTransformType.YEAR)
.build());
@@ -113,7 +113,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("timestamp_month")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
.build())
.transformType(PartitionTransformType.MONTH)
.build());
@@ -131,7 +131,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("timestamp_day")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
.build())
.transformType(PartitionTransformType.DAY)
.build());
@@ -149,7 +149,7 @@ public class TestIcebergPartitionSpecExtractor {
.sourceField(
OneField.builder()
.name("timestamp_hour")
- .schema(OneSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
+ .schema(InternalSchema.builder().dataType(OneType.TIMESTAMP_NTZ).build())
.build())
.transformType(PartitionTransformType.HOUR)
.build());
@@ -168,7 +168,7 @@ public class TestIcebergPartitionSpecExtractor {
OneField.builder()
.name("nested")
.parentPath("data")
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build())
.transformType(PartitionTransformType.VALUE)
.build());
@@ -205,19 +205,19 @@ public class TestIcebergPartitionSpecExtractor {
PartitionSpec icePartitionSpec =
PartitionSpec.builderFor(iceSchema).identity("key_string").build();
- OneSchema irSchema =
- OneSchema.builder()
+ InternalSchema irSchema =
+ InternalSchema.builder()
.name("test_schema")
.fields(
Arrays.asList(
OneField.builder()
.name("data_int")
- .schema(OneSchema.builder().dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().dataType(OneType.INT).build())
.build(),
OneField.builder()
.name("key_string")
.fieldId(1)
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build()))
.build();
@@ -243,20 +243,20 @@ public class TestIcebergPartitionSpecExtractor {
PartitionSpec icePartitionSpec =
PartitionSpec.builderFor(iceSchema).identity("key_string").year("key_year").build();
- OneSchema irSchema =
- OneSchema.builder()
+ InternalSchema irSchema =
+ InternalSchema.builder()
.name("test_schema")
.fields(
Arrays.asList(
OneField.builder()
.name("key_year")
.fieldId(10)
- .schema(OneSchema.builder().dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().dataType(OneType.DATE).build())
.build(),
OneField.builder()
.name("key_string")
.fieldId(11)
- .schema(OneSchema.builder().dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().dataType(OneType.STRING).build())
.build()))
.build();
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
index 3d0f39d2..537ef746 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
@@ -33,9 +33,9 @@ import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.types.Types;
import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.stat.PartitionValue;
import org.apache.xtable.model.stat.Range;
@@ -61,7 +61,7 @@ public class TestIcebergPartitionValueConverter {
"abc",
1614556800000L,
51 /* Iceberg represents year as diff from 1970 */);
- private static final OneSchema ONE_SCHEMA =
+ private static final InternalSchema ONE_SCHEMA =
IcebergSchemaExtractor.getInstance().fromIceberg(SCHEMA);
@Test
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaExtractor.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaExtractor.java
index 83a6dab7..94178235 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaExtractor.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSchemaExtractor.java
@@ -29,8 +29,8 @@ import org.junit.jupiter.api.Test;
import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
public class TestIcebergSchemaExtractor {
@@ -42,16 +42,16 @@ public class TestIcebergSchemaExtractor {
public void testPrimitiveTypes() {
int precision = 10;
int scale = 5;
- Map<OneSchema.MetadataKey, Object> doubleMetadata = new HashMap<>();
- doubleMetadata.put(OneSchema.MetadataKey.DECIMAL_PRECISION, precision);
- doubleMetadata.put(OneSchema.MetadataKey.DECIMAL_SCALE, scale);
+ Map<InternalSchema.MetadataKey, Object> doubleMetadata = new HashMap<>();
+ doubleMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, precision);
+ doubleMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, scale);
int fixedSize = 8;
- Map<OneSchema.MetadataKey, Object> fixedMetadata =
- Collections.singletonMap(OneSchema.MetadataKey.FIXED_BYTES_SIZE, fixedSize);
+ Map<InternalSchema.MetadataKey, Object> fixedMetadata =
+ Collections.singletonMap(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, fixedSize);
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.isNullable(false)
.name("record")
@@ -60,7 +60,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(false)
@@ -70,7 +70,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalBoolean")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("boolean")
.dataType(OneType.BOOLEAN)
.isNullable(true)
@@ -81,7 +81,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -91,7 +91,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalInt")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(true)
@@ -102,7 +102,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(false)
@@ -112,7 +112,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalLong")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.LONG)
.isNullable(true)
@@ -123,7 +123,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -133,7 +133,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalDouble")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(true)
@@ -144,7 +144,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(false)
@@ -154,7 +154,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalFloat")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("float")
.dataType(OneType.FLOAT)
.isNullable(true)
@@ -165,7 +165,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -175,7 +175,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalString")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -186,7 +186,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(false)
@@ -196,7 +196,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalBytes")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("binary")
.dataType(OneType.BYTES)
.isNullable(true)
@@ -207,7 +207,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredDate")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("date")
.dataType(OneType.DATE)
.isNullable(false)
@@ -217,7 +217,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalDate")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("date")
.dataType(OneType.DATE)
.isNullable(true)
@@ -228,7 +228,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed")
.dataType(OneType.FIXED)
.isNullable(false)
@@ -239,7 +239,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalFixed")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("fixed")
.dataType(OneType.FIXED)
.isNullable(true)
@@ -251,7 +251,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredDecimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("decimal")
.dataType(OneType.DECIMAL)
.isNullable(false)
@@ -262,7 +262,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalDecimal")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("decimal")
.dataType(OneType.DECIMAL)
.isNullable(true)
@@ -299,27 +299,29 @@ public class TestIcebergSchemaExtractor {
30, "optionalDecimal", Types.DecimalType.of(precision, scale)));
Assertions.assertTrue(
- icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(oneSchemaRepresentation)));
- assertEquals(oneSchemaRepresentation, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
+ icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(internalSchema)));
+ assertEquals(internalSchema, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
}
@Test
public void testEnums() {
// there are no enums in iceberg so we convert them to string
- Map<OneSchema.MetadataKey, Object> requiredEnumMetadata =
- Collections.singletonMap(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
- Map<OneSchema.MetadataKey, Object> optionalEnumMetadata =
- Collections.singletonMap(OneSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
+ Map<InternalSchema.MetadataKey, Object> requiredEnumMetadata =
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("ONE", "TWO"));
+ Map<InternalSchema.MetadataKey, Object> optionalEnumMetadata =
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.ENUM_VALUES, Arrays.asList("THREE", "FOUR"));
- OneSchema schemaWithEnums =
- OneSchema.builder()
+ InternalSchema schemaWithEnums =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.fields(
Arrays.asList(
OneField.builder()
.name("requiredEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("REQUIRED_ENUM")
.dataType(OneType.ENUM)
.isNullable(false)
@@ -330,7 +332,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalEnum")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("OPTIONAL_ENUM")
.dataType(OneType.ENUM)
.isNullable(true)
@@ -356,10 +358,10 @@ public class TestIcebergSchemaExtractor {
Types.NestedField.optional(2, "optionalUuid", Types.UUIDType.get()));
int fixedSize = 16;
- Map<OneSchema.MetadataKey, Object> fixedMetadata = new HashMap<>();
- fixedMetadata.put(OneSchema.MetadataKey.FIXED_BYTES_SIZE, fixedSize);
- OneSchema expectedSchema =
- OneSchema.builder()
+ Map<InternalSchema.MetadataKey, Object> fixedMetadata = new HashMap<>();
+ fixedMetadata.put(InternalSchema.MetadataKey.FIXED_BYTES_SIZE, fixedSize);
+ InternalSchema expectedSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("record")
.fields(
@@ -368,7 +370,7 @@ public class TestIcebergSchemaExtractor {
.name("requiredUuid")
.fieldId(1)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("uuid")
.dataType(OneType.FIXED)
.isNullable(false)
@@ -379,7 +381,7 @@ public class TestIcebergSchemaExtractor {
.name("optionalUuid")
.fieldId(2)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("uuid")
.dataType(OneType.FIXED)
.isNullable(true)
@@ -393,16 +395,16 @@ public class TestIcebergSchemaExtractor {
@Test
public void testTimestamps() {
- Map<OneSchema.MetadataKey, Object> millisTimestamp =
+ Map<InternalSchema.MetadataKey, Object> millisTimestamp =
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MILLIS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MILLIS);
- Map<OneSchema.MetadataKey, Object> microsTimestamp =
+ Map<InternalSchema.MetadataKey, Object> microsTimestamp =
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION, OneSchema.MetadataValue.MICROS);
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
- OneSchema irSchema =
- OneSchema.builder()
+ InternalSchema irSchema =
+ InternalSchema.builder()
.name("record")
.dataType(OneType.RECORD)
.fields(
@@ -410,7 +412,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredTimestampMillis")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(millisTimestamp)
@@ -420,7 +422,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalTimestampMillis")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(millisTimestamp)
@@ -431,7 +433,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredTimestampNtzMillis")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(false)
@@ -441,7 +443,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalTimestampNtzMillis")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(millisTimestamp)
@@ -452,7 +454,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredTimestampMicros")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(microsTimestamp)
@@ -462,7 +464,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalTimestampMicros")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(microsTimestamp)
@@ -473,7 +475,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("requiredTimestampNtzMicros")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.isNullable(false)
@@ -483,7 +485,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("optionalTimestampNtzMicros")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestampNtz")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(microsTimestamp)
@@ -518,8 +520,8 @@ public class TestIcebergSchemaExtractor {
6, "requiredTimestampWithoutZone", Types.TimestampType.withoutZone()),
Types.NestedField.optional(
7, "optionalTimestampWithoutZone", Types.TimestampType.withoutZone()));
- OneSchema expectedIrSchema =
- OneSchema.builder()
+ InternalSchema expectedIrSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("record")
.fields(
@@ -528,7 +530,7 @@ public class TestIcebergSchemaExtractor {
.name("requiredTimestampWithZone")
.fieldId(4)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(microsTimestamp)
@@ -539,7 +541,7 @@ public class TestIcebergSchemaExtractor {
.name("optionalTimestampWithZone")
.fieldId(5)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP)
.metadata(microsTimestamp)
@@ -551,7 +553,7 @@ public class TestIcebergSchemaExtractor {
.name("requiredTimestampWithoutZone")
.fieldId(6)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(microsTimestamp)
@@ -562,7 +564,7 @@ public class TestIcebergSchemaExtractor {
.name("optionalTimestampWithoutZone")
.fieldId(7)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("timestamp")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(microsTimestamp)
@@ -576,8 +578,8 @@ public class TestIcebergSchemaExtractor {
@Test
public void testMaps() {
- OneSchema recordMapElementSchema =
- OneSchema.builder()
+ InternalSchema recordMapElementSchema =
+ InternalSchema.builder()
.name("struct")
.isNullable(true)
.fields(
@@ -587,7 +589,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("recordMap._one_field_value")
.fieldId(7)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -598,7 +600,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("recordMap._one_field_value")
.fieldId(8)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -607,8 +609,8 @@ public class TestIcebergSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("record")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -618,7 +620,7 @@ public class TestIcebergSchemaExtractor {
.name("intMap")
.fieldId(1)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(false)
.dataType(OneType.MAP)
@@ -629,7 +631,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("intMap")
.fieldId(3)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(false)
@@ -640,7 +642,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("intMap")
.fieldId(4)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -652,7 +654,7 @@ public class TestIcebergSchemaExtractor {
.name("recordMap")
.fieldId(2)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(true)
.dataType(OneType.MAP)
@@ -663,7 +665,7 @@ public class TestIcebergSchemaExtractor {
.fieldId(5)
.parentPath("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -698,14 +700,14 @@ public class TestIcebergSchemaExtractor {
Types.NestedField.optional(8, "optionalString", Types.StringType.get())))));
Assertions.assertTrue(
- icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(oneSchemaRepresentation)));
- assertEquals(oneSchemaRepresentation, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
+ icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(internalSchema)));
+ assertEquals(internalSchema, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
}
@Test
public void testLists() {
- OneSchema recordListElementSchema =
- OneSchema.builder()
+ InternalSchema recordListElementSchema =
+ InternalSchema.builder()
.name("struct")
.isNullable(true)
.fields(
@@ -715,7 +717,7 @@ public class TestIcebergSchemaExtractor {
.fieldId(5)
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -726,7 +728,7 @@ public class TestIcebergSchemaExtractor {
.fieldId(6)
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -735,8 +737,8 @@ public class TestIcebergSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("record")
.isNullable(false)
@@ -746,7 +748,7 @@ public class TestIcebergSchemaExtractor {
.name("intList")
.fieldId(1)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("list")
.isNullable(false)
.dataType(OneType.LIST)
@@ -757,7 +759,7 @@ public class TestIcebergSchemaExtractor {
.fieldId(3)
.parentPath("intList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(false)
@@ -769,7 +771,7 @@ public class TestIcebergSchemaExtractor {
.name("recordList")
.fieldId(2)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("list")
.isNullable(true)
.dataType(OneType.LIST)
@@ -800,14 +802,14 @@ public class TestIcebergSchemaExtractor {
Types.NestedField.optional(6, "optionalString", Types.StringType.get())))));
Assertions.assertTrue(
- icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(oneSchemaRepresentation)));
- assertEquals(oneSchemaRepresentation, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
+ icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(internalSchema)));
+ assertEquals(internalSchema, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
}
@Test
public void testNestedRecords() {
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("record")
.isNullable(false)
@@ -818,7 +820,7 @@ public class TestIcebergSchemaExtractor {
.defaultValue(OneField.Constants.NULL_DEFAULT_VALUE)
.fieldId(1)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(true)
@@ -829,7 +831,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("nestedOne")
.fieldId(2)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("integer")
.dataType(OneType.INT)
.isNullable(true)
@@ -841,7 +843,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("nestedOne")
.fieldId(3)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -852,7 +854,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("nestedOne")
.fieldId(4)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("struct")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -863,7 +865,7 @@ public class TestIcebergSchemaExtractor {
.parentPath("nestedOne.nestedTwo")
.fieldId(5)
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -893,14 +895,14 @@ public class TestIcebergSchemaExtractor {
Types.NestedField.optional(
5, "doublyNestedString", Types.StringType.get()))))));
Assertions.assertTrue(
- icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(oneSchemaRepresentation)));
- assertEquals(oneSchemaRepresentation, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
+ icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(internalSchema)));
+ assertEquals(internalSchema, SCHEMA_EXTRACTOR.fromIceberg(icebergRepresentation));
}
@Test
public void testToIcebergWithNoFieldIdsSet() {
- OneSchema recordListElementSchema =
- OneSchema.builder()
+ InternalSchema recordListElementSchema =
+ InternalSchema.builder()
.name("element")
.isNullable(true)
.fields(
@@ -909,7 +911,7 @@ public class TestIcebergSchemaExtractor {
.name("requiredDouble")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -919,7 +921,7 @@ public class TestIcebergSchemaExtractor {
.name("optionalString")
.parentPath("recordList._one_field_element")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -928,8 +930,8 @@ public class TestIcebergSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema recordMapElementSchema =
- OneSchema.builder()
+ InternalSchema recordMapElementSchema =
+ InternalSchema.builder()
.name("element")
.isNullable(true)
.fields(
@@ -938,7 +940,7 @@ public class TestIcebergSchemaExtractor {
.name("requiredDouble")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("double")
.dataType(OneType.DOUBLE)
.isNullable(false)
@@ -948,7 +950,7 @@ public class TestIcebergSchemaExtractor {
.name("optionalString")
.parentPath("recordMap._one_field_value")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.isNullable(true)
@@ -957,8 +959,8 @@ public class TestIcebergSchemaExtractor {
.build()))
.dataType(OneType.RECORD)
.build();
- OneSchema oneSchemaRepresentation =
- OneSchema.builder()
+ InternalSchema internalSchema =
+ InternalSchema.builder()
.name("testRecord")
.dataType(OneType.RECORD)
.isNullable(false)
@@ -967,7 +969,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("recordList")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.isNullable(true)
.dataType(OneType.LIST)
@@ -984,7 +986,7 @@ public class TestIcebergSchemaExtractor {
OneField.builder()
.name("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.isNullable(true)
.dataType(OneType.MAP)
@@ -994,7 +996,7 @@ public class TestIcebergSchemaExtractor {
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("recordMap")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map_key")
.dataType(OneType.INT)
.isNullable(false)
@@ -1033,6 +1035,6 @@ public class TestIcebergSchemaExtractor {
Types.NestedField.optional(9, "optionalString", Types.StringType.get())))));
Assertions.assertTrue(
- icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(oneSchemaRepresentation)));
+ icebergRepresentation.sameSchema(SCHEMA_EXTRACTOR.toIceberg(internalSchema)));
}
}
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
index f8076209..f44cf29d 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
@@ -56,8 +56,8 @@ import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.schema.SchemaVersion;
@@ -136,10 +136,10 @@ class TestIcebergSourceClient {
SchemaCatalog schemaCatalog = spyClient.getSchemaCatalog(null, iceCurrentSnapshot);
Assertions.assertNotNull(schemaCatalog);
- Map<SchemaVersion, OneSchema> schemas = schemaCatalog.getSchemas();
+ Map<SchemaVersion, InternalSchema> schemas = schemaCatalog.getSchemas();
assertEquals(1, schemas.size());
SchemaVersion expectedSchemaVersion = new SchemaVersion(iceCurrentSnapshot.schemaId(), "");
- OneSchema irSchemaOfCommit = schemas.get(expectedSchemaVersion);
+ InternalSchema irSchemaOfCommit = schemas.get(expectedSchemaVersion);
Assertions.assertNotNull(irSchemaOfCommit);
validateSchema(irSchemaOfCommit, catalogSales.schemas().get(iceCurrentSnapshot.schemaId()));
}
@@ -347,7 +347,7 @@ class TestIcebergSourceClient {
assertEquals(removedFiles, tableChange.getFilesDiff().getFilesRemoved().size());
}
- private void validateSchema(OneSchema readSchema, Schema expectedSchema) {
+ private void validateSchema(InternalSchema readSchema, Schema expectedSchema) {
IcebergSchemaExtractor schemaExtractor = IcebergSchemaExtractor.getInstance();
Schema result = schemaExtractor.toIceberg(readSchema);
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
index 9fc8ef11..73062621 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
@@ -86,9 +86,9 @@ import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.OneSnapshot;
import org.apache.xtable.model.OneTable;
import org.apache.xtable.model.OneTableMetadata;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
import org.apache.xtable.model.schema.OnePartitionField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.schema.SchemaCatalog;
@@ -125,8 +125,8 @@ public class TestIcebergSync {
Mockito.mock(IcebergColumnStatsConverter.class);
private IcebergClient icebergClient;
- private final OneSchema oneSchema =
- OneSchema.builder()
+ private final InternalSchema internalSchema =
+ InternalSchema.builder()
.dataType(OneType.RECORD)
.name("parent")
.fields(
@@ -134,20 +134,23 @@ public class TestIcebergSync {
OneField.builder()
.name("timestamp_field")
.schema(
- OneSchema.builder().name("long").dataType(OneType.TIMESTAMP_NTZ).build())
+ InternalSchema.builder()
+ .name("long")
+ .dataType(OneType.TIMESTAMP_NTZ)
+ .build())
.build(),
OneField.builder()
.name("date_field")
- .schema(OneSchema.builder().name("int").dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.DATE).build())
.build(),
OneField.builder()
.name("group_id")
- .schema(OneSchema.builder().name("int").dataType(OneType.INT).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.INT).build())
.build(),
OneField.builder()
.name("record")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("nested")
.dataType(OneType.RECORD)
.fields(
@@ -156,7 +159,7 @@ public class TestIcebergSync {
.name("string_field")
.parentPath("record")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("string")
.dataType(OneType.STRING)
.build())
@@ -206,21 +209,21 @@ public class TestIcebergSync {
@Test
public void testCreateSnapshotControlFlow() throws Exception {
// Test two iterations of the iceberg snapshot flow
- List<OneField> fields2 = new ArrayList<>(oneSchema.getFields());
+ List<OneField> fields2 = new ArrayList<>(internalSchema.getFields());
fields2.add(
OneField.builder()
.name("long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build());
- OneSchema schema2 = oneSchema.toBuilder().fields(fields2).build();
+ InternalSchema schema2 = internalSchema.toBuilder().fields(fields2).build();
List<Types.NestedField> fields = new ArrayList<>(icebergSchema.columns());
fields.add(Types.NestedField.of(6, false, "long_field", Types.LongType.get()));
Schema icebergSchema2 = new Schema(fields);
- OneTable table1 = getOneTable(tableName, basePath, oneSchema, null, LAST_COMMIT_TIME);
+ OneTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
OneTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion1 = new SchemaVersion(1, "");
- schemas.put(schemaVersion1, oneSchema);
+ schemas.put(schemaVersion1, internalSchema);
SchemaVersion schemaVersion2 = new SchemaVersion(2, "");
schemas.put(schemaVersion2, schema2);
@@ -229,7 +232,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion2, 3, Collections.emptyList());
OneSnapshot snapshot1 = buildSnapshot(table1, schemas, dataFile1, dataFile2);
OneSnapshot snapshot2 = buildSnapshot(table2, schemas, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
when(mockSchemaExtractor.toIceberg(schema2)).thenReturn(icebergSchema2);
ArgumentCaptor<Schema> partitionSpecSchemaArgumentCaptor =
ArgumentCaptor.forClass(Schema.class);
@@ -301,22 +304,22 @@ public class TestIcebergSync {
@Test
public void testIncompleteWriteRollback() throws Exception {
- List<OneField> fields2 = new ArrayList<>(oneSchema.getFields());
+ List<OneField> fields2 = new ArrayList<>(internalSchema.getFields());
fields2.add(
OneField.builder()
.name("long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build());
- OneSchema schema2 = oneSchema.toBuilder().fields(fields2).build();
+ InternalSchema schema2 = internalSchema.toBuilder().fields(fields2).build();
List<Types.NestedField> fields = new ArrayList<>(icebergSchema.columns());
fields.add(Types.NestedField.of(6, false, "long_field", Types.LongType.get()));
Schema icebergSchema2 = new Schema(fields);
- OneTable table1 = getOneTable(tableName, basePath, oneSchema, null, LAST_COMMIT_TIME);
+ OneTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
OneTable table2 =
getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME.plusMillis(100000L));
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion1 = new SchemaVersion(1, "");
- schemas.put(schemaVersion1, oneSchema);
+ schemas.put(schemaVersion1, internalSchema);
SchemaVersion schemaVersion2 = new SchemaVersion(2, "");
schemas.put(schemaVersion2, schema2);
@@ -327,7 +330,7 @@ public class TestIcebergSync {
OneSnapshot snapshot1 = buildSnapshot(table1, schemas, dataFile1, dataFile2);
OneSnapshot snapshot2 = buildSnapshot(table2, schemas, dataFile2, dataFile3);
OneSnapshot snapshot3 = buildSnapshot(table2, schemas, dataFile3, dataFile4);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
when(mockSchemaExtractor.toIceberg(schema2)).thenReturn(icebergSchema2);
ArgumentCaptor<Schema> partitionSpecSchemaArgumentCaptor =
ArgumentCaptor.forClass(Schema.class);
@@ -370,7 +373,7 @@ public class TestIcebergSync {
OnePartitionField partitionField =
OnePartitionField.builder()
.sourceField(
- SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "timestamp_field"))
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, "timestamp_field"))
.transformType(PartitionTransformType.DAY)
.build();
@@ -378,12 +381,12 @@ public class TestIcebergSync {
getOneTable(
tableName,
basePath,
- oneSchema,
+ internalSchema,
Collections.singletonList(partitionField),
LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
- schemas.put(schemaVersion, oneSchema);
+ schemas.put(schemaVersion, internalSchema);
List<PartitionValue> partitionValues1 =
Collections.singletonList(
@@ -402,7 +405,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion, 3, partitionValues2);
OneSnapshot snapshot = buildSnapshot(table, schemas, dataFile1, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema))
+ when(mockSchemaExtractor.toIceberg(internalSchema))
.thenReturn(icebergSchema)
.thenReturn(icebergSchema);
PartitionSpec partitionSpec =
@@ -434,7 +437,8 @@ public class TestIcebergSync {
// test partition filtering
OnePartitionField partitionField =
OnePartitionField.builder()
- .sourceField(SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "date_field"))
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, "date_field"))
.transformType(PartitionTransformType.DAY)
.build();
@@ -442,12 +446,12 @@ public class TestIcebergSync {
getOneTable(
tableName,
basePath,
- oneSchema,
+ internalSchema,
Collections.singletonList(partitionField),
LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
- schemas.put(schemaVersion, oneSchema);
+ schemas.put(schemaVersion, internalSchema);
List<PartitionValue> partitionValues1 =
Collections.singletonList(
@@ -466,7 +470,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion, 3, partitionValues2);
OneSnapshot snapshot = buildSnapshot(table, schemas, dataFile1, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
PartitionSpec partitionSpec =
PartitionSpec.builderFor(icebergSchema)
.day(partitionField.getSourceField().getName())
@@ -495,7 +499,8 @@ public class TestIcebergSync {
// test partition filtering
OnePartitionField partitionField =
OnePartitionField.builder()
- .sourceField(SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "group_id"))
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, "group_id"))
.transformType(PartitionTransformType.VALUE)
.build();
@@ -503,12 +508,12 @@ public class TestIcebergSync {
getOneTable(
tableName,
basePath,
- oneSchema,
+ internalSchema,
Collections.singletonList(partitionField),
LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
- schemas.put(schemaVersion, oneSchema);
+ schemas.put(schemaVersion, internalSchema);
List<PartitionValue> partitionValues1 =
Collections.singletonList(
@@ -521,7 +526,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion, 3, partitionValues2);
OneSnapshot snapshot = buildSnapshot(table, schemas, dataFile1, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
PartitionSpec partitionSpec =
PartitionSpec.builderFor(icebergSchema)
.identity(partitionField.getSourceField().getName())
@@ -550,13 +555,14 @@ public class TestIcebergSync {
// test partition filtering
OnePartitionField partitionField1 =
OnePartitionField.builder()
- .sourceField(SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "group_id"))
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, "group_id"))
.transformType(PartitionTransformType.VALUE)
.build();
OnePartitionField partitionField2 =
OnePartitionField.builder()
.sourceField(
- SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "timestamp_field"))
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, "timestamp_field"))
.transformType(PartitionTransformType.DAY)
.build();
@@ -564,12 +570,12 @@ public class TestIcebergSync {
getOneTable(
tableName,
basePath,
- oneSchema,
+ internalSchema,
Arrays.asList(partitionField1, partitionField2),
LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
- schemas.put(schemaVersion, oneSchema);
+ schemas.put(schemaVersion, internalSchema);
List<PartitionValue> partitionValues1 =
Arrays.asList(
@@ -597,7 +603,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion, 3, partitionValues3);
OneSnapshot snapshot = buildSnapshot(table, schemas, dataFile1, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
PartitionSpec partitionSpec =
PartitionSpec.builderFor(icebergSchema)
.identity(partitionField1.getSourceField().getName())
@@ -632,7 +638,8 @@ public class TestIcebergSync {
OnePartitionField partitionField =
OnePartitionField.builder()
.sourceField(
- SchemaFieldFinder.getInstance().findFieldByPath(oneSchema, "record.string_field"))
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, "record.string_field"))
.transformType(PartitionTransformType.VALUE)
.build();
@@ -640,12 +647,12 @@ public class TestIcebergSync {
getOneTable(
tableName,
basePath,
- oneSchema,
+ internalSchema,
Collections.singletonList(partitionField),
LAST_COMMIT_TIME);
- Map<SchemaVersion, OneSchema> schemas = new HashMap<>();
+ Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
- schemas.put(schemaVersion, oneSchema);
+ schemas.put(schemaVersion, internalSchema);
List<PartitionValue> partitionValues1 =
Collections.singletonList(
@@ -664,7 +671,7 @@ public class TestIcebergSync {
InternalDataFile dataFile3 = getDataFile(schemaVersion, 3, partitionValues2);
OneSnapshot snapshot = buildSnapshot(table, schemas, dataFile1, dataFile2, dataFile3);
- when(mockSchemaExtractor.toIceberg(oneSchema)).thenReturn(icebergSchema);
+ when(mockSchemaExtractor.toIceberg(internalSchema)).thenReturn(icebergSchema);
PartitionSpec partitionSpec =
PartitionSpec.builderFor(icebergSchema)
.identity(partitionField.getSourceField().getPath())
@@ -687,7 +694,7 @@ public class TestIcebergSync {
}
private OneSnapshot buildSnapshot(
- OneTable table, Map<SchemaVersion, OneSchema> schemas, InternalDataFile... dataFiles) {
+ OneTable table, Map<SchemaVersion, InternalSchema> schemas, InternalDataFile... dataFiles) {
return OneSnapshot.builder()
.table(table)
.schemaCatalog(SchemaCatalog.builder().schemas(schemas).build())
@@ -712,7 +719,7 @@ public class TestIcebergSync {
private OneTable getOneTable(
String tableName,
Path basePath,
- OneSchema schema,
+ InternalSchema schema,
List<OnePartitionField> partitionFields,
Instant lastCommitTime) {
return OneTable.builder()
diff --git a/core/src/test/java/org/apache/xtable/testutil/ColumnStatMapUtil.java b/core/src/test/java/org/apache/xtable/testutil/ColumnStatMapUtil.java
index 4c3d6c81..7529aaa4 100644
--- a/core/src/test/java/org/apache/xtable/testutil/ColumnStatMapUtil.java
+++ b/core/src/test/java/org/apache/xtable/testutil/ColumnStatMapUtil.java
@@ -26,8 +26,8 @@ import java.util.List;
import lombok.AccessLevel;
import lombok.NoArgsConstructor;
+import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.OneField;
-import org.apache.xtable.model.schema.OneSchema;
import org.apache.xtable.model.schema.OneType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.stat.Range;
@@ -37,75 +37,75 @@ public class ColumnStatMapUtil {
private static final OneField LONG_FIELD =
OneField.builder()
.name("long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build();
private static final OneField STRING_FIELD =
OneField.builder()
.name("string_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
private static final OneField NULL_STRING_FIELD =
OneField.builder()
.name("null_string_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
private static final OneField TIMESTAMP_FIELD =
OneField.builder()
.name("timestamp_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION,
- OneSchema.MetadataValue.MILLIS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MILLIS))
.build())
.build();
private static final OneField TIMESTAMP_MICROS_FIELD =
OneField.builder()
.name("timestamp_micros_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION,
- OneSchema.MetadataValue.MICROS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
.build())
.build();
private static final OneField LOCAL_TIMESTAMP_FIELD =
OneField.builder()
.name("local_timestamp_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("long")
.dataType(OneType.TIMESTAMP_NTZ)
.metadata(
Collections.singletonMap(
- OneSchema.MetadataKey.TIMESTAMP_PRECISION,
- OneSchema.MetadataValue.MILLIS))
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MILLIS))
.build())
.build();
private static final OneField DATE_FIELD =
OneField.builder()
.name("date_field")
- .schema(OneSchema.builder().name("int").dataType(OneType.DATE).build())
+ .schema(InternalSchema.builder().name("int").dataType(OneType.DATE).build())
.build();
private static final OneField ARRAY_LONG_FIELD_ELEMENT =
OneField.builder()
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("array_long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build();
private static final OneField ARRAY_LONG_FIELD =
OneField.builder()
.name("array_long_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.dataType(OneType.LIST)
.fields(Collections.singletonList(ARRAY_LONG_FIELD_ELEMENT))
@@ -116,19 +116,19 @@ public class ColumnStatMapUtil {
OneField.builder()
.name(OneField.Constants.MAP_KEY_FIELD_NAME)
.parentPath("map_string_long_field")
- .schema(OneSchema.builder().name("map_key").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("map_key").dataType(OneType.STRING).build())
.build();
private static final OneField MAP_VALUE_LONG_FIELD =
OneField.builder()
.name(OneField.Constants.MAP_VALUE_FIELD_NAME)
.parentPath("map_string_long_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build();
private static final OneField MAP_STRING_LONG_FIELD =
OneField.builder()
.name("map_string_long_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("map")
.dataType(OneType.MAP)
.fields(Arrays.asList(MAP_KEY_STRING_FIELD, MAP_VALUE_LONG_FIELD))
@@ -139,14 +139,14 @@ public class ColumnStatMapUtil {
OneField.builder()
.name(OneField.Constants.ARRAY_ELEMENT_FIELD_NAME)
.parentPath("nested_struct_field.array_string_field")
- .schema(OneSchema.builder().name("string").dataType(OneType.STRING).build())
+ .schema(InternalSchema.builder().name("string").dataType(OneType.STRING).build())
.build();
private static final OneField NESTED_ARRAY_STRING_FIELD =
OneField.builder()
.name("array_string_field")
.parentPath("nested_struct_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("array")
.dataType(OneType.LIST)
.fields(Collections.singletonList(NESTED_ARRAY_STRING_FIELD_ELEMENT))
@@ -157,14 +157,14 @@ public class ColumnStatMapUtil {
OneField.builder()
.name("nested_long_field")
.parentPath("nested_struct_field")
- .schema(OneSchema.builder().name("long").dataType(OneType.LONG).build())
+ .schema(InternalSchema.builder().name("long").dataType(OneType.LONG).build())
.build();
private static final OneField NESTED_STRUCT_FIELD =
OneField.builder()
.name("nested_struct_field")
.schema(
- OneSchema.builder()
+ InternalSchema.builder()
.name("nested_struct_field")
.dataType(OneType.RECORD)
.fields(Arrays.asList(NESTED_ARRAY_STRING_FIELD, NESTED_LONG_FIELD))
@@ -174,23 +174,23 @@ public class ColumnStatMapUtil {
private static final OneField DECIMAL_FIELD =
OneField.builder()
.name("decimal_field")
- .schema(OneSchema.builder().name("decimal").dataType(OneType.DECIMAL).build())
+ .schema(InternalSchema.builder().name("decimal").dataType(OneType.DECIMAL).build())
.build();
private static final OneField FLOAT_FIELD =
OneField.builder()
.name("float_field")
- .schema(OneSchema.builder().name("float").dataType(OneType.FLOAT).build())
+ .schema(InternalSchema.builder().name("float").dataType(OneType.FLOAT).build())
.build();
private static final OneField DOUBLE_FIELD =
OneField.builder()
.name("double_field")
- .schema(OneSchema.builder().name("double").dataType(OneType.DOUBLE).build())
+ .schema(InternalSchema.builder().name("double").dataType(OneType.DOUBLE).build())
.build();
- public static OneSchema getSchema() {
- return OneSchema.builder()
+ public static InternalSchema getSchema() {
+ return InternalSchema.builder()
.name("record")
.dataType(OneType.RECORD)
.fields(