You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@xtable.apache.org by as...@apache.org on 2024/03/28 20:50:40 UTC
(incubator-xtable) 04/04: Refactor: rename OneTable to InternalTable
This is an automated email from the ASF dual-hosted git repository.
ashvin pushed a commit to branch 396-fix-prefix-in-classnames
in repository https://gitbox.apache.org/repos/asf/incubator-xtable.git
commit 8e50c315c161696a2335edc46b2ba134c94a4a68
Author: Ashvin Agrawal <as...@apache.org>
AuthorDate: Thu Mar 28 13:30:00 2024 -0700
Refactor: rename OneTable to InternalTable
---
.../org/apache/xtable/model/InternalSnapshot.java | 6 ++--
.../model/{OneTable.java => InternalTable.java} | 2 +-
.../java/org/apache/xtable/model/TableChange.java | 4 +--
.../org/apache/xtable/model/storage/FilesDiff.java | 2 +-
.../{OneFileGroup.java => PartitionFileGroup.java} | 8 ++---
.../xtable/model/validation/ValidationChecker.java | 6 ++--
.../apache/xtable/spi/extractor/SourceClient.java | 8 ++---
.../xtable/spi/extractor/TableExtractor.java | 6 ++--
.../apache/xtable/spi/sync/TableFormatSync.java | 10 +++---
.../org/apache/xtable/spi/sync/TargetClient.java | 8 ++---
.../apache/xtable/model/storage/TestFilesDiff.java | 5 +--
.../spi/extractor/TestExtractFromSource.java | 15 +++++----
.../xtable/spi/sync/TestTableFormatSync.java | 32 ++++++++++---------
.../org/apache/xtable/client/OneTableClient.java | 2 +-
.../apache/xtable/constants/OneTableConstants.java | 2 +-
.../java/org/apache/xtable/delta/DeltaClient.java | 8 ++---
.../delta/DeltaDataFileUpdatesExtractor.java | 6 ++--
.../apache/xtable/delta/DeltaSchemaExtractor.java | 2 +-
.../org/apache/xtable/delta/DeltaSourceClient.java | 16 +++++-----
.../apache/xtable/delta/DeltaTableExtractor.java | 10 +++---
.../xtable/hudi/BaseFileUpdatesExtractor.java | 6 ++--
.../java/org/apache/xtable/hudi/HudiClient.java | 10 +++---
.../apache/xtable/hudi/HudiDataFileExtractor.java | 16 +++++-----
.../xtable/hudi/HudiSchemaCatalogExtractor.java | 4 +--
.../org/apache/xtable/hudi/HudiTableExtractor.java | 10 +++---
.../org/apache/xtable/hudi/HudiTableManager.java | 6 ++--
.../org/apache/xtable/hudi/HudiTargetClient.java | 8 ++---
.../org/apache/xtable/iceberg/IcebergClient.java | 25 ++++++++-------
.../iceberg/IcebergColumnStatsConverter.java | 2 +-
.../xtable/iceberg/IcebergDataFileExtractor.java | 6 ++--
.../xtable/iceberg/IcebergDataFileUpdatesSync.java | 8 ++---
.../iceberg/IcebergPartitionValueConverter.java | 12 +++----
.../apache/xtable/iceberg/IcebergSourceClient.java | 24 +++++++-------
.../org/apache/xtable/ValidationTestHelper.java | 10 +++---
.../apache/xtable/client/TestOneTableClient.java | 24 +++++++-------
.../apache/xtable/delta/ITDeltaSourceClient.java | 22 ++++++-------
.../org/apache/xtable/delta/TestDeltaSync.java | 22 ++++++-------
.../org/apache/xtable/hudi/ITHudiTargetClient.java | 24 +++++++-------
.../xtable/hudi/TestBaseFileUpdatesExtractor.java | 18 +++++------
.../apache/xtable/hudi/TestHudiTableManager.java | 6 ++--
.../apache/xtable/hudi/TestHudiTargetClient.java | 14 +++++---
.../TestIcebergPartitionValueConverter.java | 8 ++---
.../xtable/iceberg/TestIcebergSourceClient.java | 37 +++++++++++-----------
.../org/apache/xtable/iceberg/TestIcebergSync.java | 34 ++++++++++----------
.../apache/xtable/hudi/sync/OneTableSyncTool.java | 7 ++--
.../xtable/hudi/sync/TestOneTableSyncTool.java | 3 +-
.../java/org/apache/xtable/utilities/RunSync.java | 2 +-
.../main/resources/onetable-client-defaults.yaml | 2 +-
48 files changed, 276 insertions(+), 252 deletions(-)
diff --git a/api/src/main/java/org/apache/xtable/model/InternalSnapshot.java b/api/src/main/java/org/apache/xtable/model/InternalSnapshot.java
index 6763f2c0..4c234922 100644
--- a/api/src/main/java/org/apache/xtable/model/InternalSnapshot.java
+++ b/api/src/main/java/org/apache/xtable/model/InternalSnapshot.java
@@ -26,7 +26,7 @@ import lombok.Builder;
import lombok.Value;
import org.apache.xtable.model.schema.SchemaCatalog;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
/**
* Snapshot represents the view of the table at a specific point in time. Snapshot captures all the
@@ -43,11 +43,11 @@ public class InternalSnapshot {
// The instant of the Snapshot
String version;
// Table reference
- OneTable table;
+ InternalTable table;
// Schema catalog referencing the written schema for each data file in the snapshot
SchemaCatalog schemaCatalog;
// Data files grouped by partition
- List<OneFileGroup> partitionedDataFiles;
+ List<PartitionFileGroup> partitionedDataFiles;
// pending commits before latest commit on the table.
@Builder.Default List<Instant> pendingCommits = Collections.emptyList();
}
diff --git a/api/src/main/java/org/apache/xtable/model/OneTable.java b/api/src/main/java/org/apache/xtable/model/InternalTable.java
similarity index 98%
rename from api/src/main/java/org/apache/xtable/model/OneTable.java
rename to api/src/main/java/org/apache/xtable/model/InternalTable.java
index b6897c3a..f7678750 100644
--- a/api/src/main/java/org/apache/xtable/model/OneTable.java
+++ b/api/src/main/java/org/apache/xtable/model/InternalTable.java
@@ -35,7 +35,7 @@ import org.apache.xtable.model.storage.DataLayoutStrategy;
*/
@Value
@Builder(toBuilder = true)
-public class OneTable {
+public class InternalTable {
// name of the table
String name;
// table format the table currently has data in
diff --git a/api/src/main/java/org/apache/xtable/model/TableChange.java b/api/src/main/java/org/apache/xtable/model/TableChange.java
index 55996770..51f0ee0b 100644
--- a/api/src/main/java/org/apache/xtable/model/TableChange.java
+++ b/api/src/main/java/org/apache/xtable/model/TableChange.java
@@ -34,6 +34,6 @@ public class TableChange {
// Change in files at the specified instant
DataFilesDiff filesDiff;
- /** The {@link OneTable} at the commit time to which this table change belongs. */
- OneTable tableAsOfChange;
+ /** The {@link InternalTable} at the commit time to which this table change belongs. */
+ InternalTable tableAsOfChange;
}
diff --git a/api/src/main/java/org/apache/xtable/model/storage/FilesDiff.java b/api/src/main/java/org/apache/xtable/model/storage/FilesDiff.java
index 0657b9f9..7d628ce4 100644
--- a/api/src/main/java/org/apache/xtable/model/storage/FilesDiff.java
+++ b/api/src/main/java/org/apache/xtable/model/storage/FilesDiff.java
@@ -93,7 +93,7 @@ public class FilesDiff<L, P> {
* @return the set of files that are added
*/
public static <P> FilesDiff<InternalDataFile, P> findNewAndRemovedFiles(
- List<OneFileGroup> latestFileGroups, Map<String, P> previousFiles) {
+ List<PartitionFileGroup> latestFileGroups, Map<String, P> previousFiles) {
Map<String, InternalDataFile> latestFiles =
latestFileGroups.stream()
.flatMap(group -> group.getFiles().stream())
diff --git a/api/src/main/java/org/apache/xtable/model/storage/OneFileGroup.java b/api/src/main/java/org/apache/xtable/model/storage/PartitionFileGroup.java
similarity index 87%
rename from api/src/main/java/org/apache/xtable/model/storage/OneFileGroup.java
rename to api/src/main/java/org/apache/xtable/model/storage/PartitionFileGroup.java
index 32c73f25..500fa1b5 100644
--- a/api/src/main/java/org/apache/xtable/model/storage/OneFileGroup.java
+++ b/api/src/main/java/org/apache/xtable/model/storage/PartitionFileGroup.java
@@ -31,21 +31,21 @@ import org.apache.xtable.model.stat.PartitionValue;
/** Represents a grouping of {@link InternalDataFile} with the same partition values. */
@Value
@Builder
-public class OneFileGroup {
+public class PartitionFileGroup {
List<PartitionValue> partitionValues;
List<InternalDataFile> files;
- public static List<OneFileGroup> fromFiles(List<InternalDataFile> files) {
+ public static List<PartitionFileGroup> fromFiles(List<InternalDataFile> files) {
return fromFiles(files.stream());
}
- public static List<OneFileGroup> fromFiles(Stream<InternalDataFile> files) {
+ public static List<PartitionFileGroup> fromFiles(Stream<InternalDataFile> files) {
Map<List<PartitionValue>, List<InternalDataFile>> filesGrouped =
files.collect(Collectors.groupingBy(InternalDataFile::getPartitionValues));
return filesGrouped.entrySet().stream()
.map(
entry ->
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.partitionValues(entry.getKey())
.files(entry.getValue())
.build())
diff --git a/api/src/main/java/org/apache/xtable/model/validation/ValidationChecker.java b/api/src/main/java/org/apache/xtable/model/validation/ValidationChecker.java
index a92ea50b..2996eacd 100644
--- a/api/src/main/java/org/apache/xtable/model/validation/ValidationChecker.java
+++ b/api/src/main/java/org/apache/xtable/model/validation/ValidationChecker.java
@@ -20,16 +20,16 @@ package org.apache.xtable.model.validation;
import java.util.Map;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.storage.TableFormat;
/**
* Interface to implement a validation checker. Runs the specified list of {@link ValidationCheck}
- * on the {@link OneTable} using the specified {@link TableFormat}
+ * on the {@link InternalTable} using the specified {@link TableFormat}
*
* @since 0.1
*/
public interface ValidationChecker {
Map<ValidationCheck, ValidationResult> validate(
- OneTable table, TableFormat tableFormat, ValidationCheck[] checks);
+ InternalTable table, TableFormat tableFormat, ValidationCheck[] checks);
}
diff --git a/api/src/main/java/org/apache/xtable/spi/extractor/SourceClient.java b/api/src/main/java/org/apache/xtable/spi/extractor/SourceClient.java
index 746ca3e3..40bd0f0d 100644
--- a/api/src/main/java/org/apache/xtable/spi/extractor/SourceClient.java
+++ b/api/src/main/java/org/apache/xtable/spi/extractor/SourceClient.java
@@ -24,7 +24,7 @@ import java.time.Instant;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.SchemaCatalog;
@@ -35,12 +35,12 @@ import org.apache.xtable.model.schema.SchemaCatalog;
*/
public interface SourceClient<COMMIT> extends Closeable {
/**
- * Extracts the {@link OneTable} definition as of the provided commit.
+ * Extracts the {@link InternalTable} definition as of the provided commit.
*
* @param commit the commit to consider for reading the table state
* @return the table definition
*/
- OneTable getTable(COMMIT commit);
+ InternalTable getTable(COMMIT commit);
/**
* Extracts the {@link SchemaCatalog} as of the provided instant.
@@ -49,7 +49,7 @@ public interface SourceClient<COMMIT> extends Closeable {
* @param commit the commit to consider for reading the schema catalog
* @return the schema catalog
*/
- SchemaCatalog getSchemaCatalog(OneTable table, COMMIT commit);
+ SchemaCatalog getSchemaCatalog(InternalTable table, COMMIT commit);
/**
* Extracts the {@link InternalSnapshot} as of latest state.
diff --git a/api/src/main/java/org/apache/xtable/spi/extractor/TableExtractor.java b/api/src/main/java/org/apache/xtable/spi/extractor/TableExtractor.java
index 854d1358..2a524fb9 100644
--- a/api/src/main/java/org/apache/xtable/spi/extractor/TableExtractor.java
+++ b/api/src/main/java/org/apache/xtable/spi/extractor/TableExtractor.java
@@ -18,13 +18,13 @@
package org.apache.xtable.spi.extractor;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
/**
- * Extracts {@link OneTable} from given {@link CLIENT}.
+ * Extracts {@link InternalTable} from given {@link CLIENT}.
*
* @param <CLIENT> Extracts canonical table model from table client.
*/
public interface TableExtractor<CLIENT> {
- OneTable table(CLIENT tableServiceClient);
+ InternalTable table(CLIENT tableServiceClient);
}
diff --git a/api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java b/api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java
index 22f0e470..869331da 100644
--- a/api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java
+++ b/api/src/main/java/org/apache/xtable/spi/sync/TableFormatSync.java
@@ -35,13 +35,13 @@ import lombok.extern.log4j.Log4j2;
import org.apache.xtable.model.IncrementalTableChanges;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.sync.SyncMode;
import org.apache.xtable.model.sync.SyncResult;
-/** Provides the functionality to sync from the OneTable format to the target format. */
+/** Provides the functionality to sync from the InternalTable format to the target format. */
@Log4j2
@NoArgsConstructor(access = AccessLevel.PRIVATE)
public class TableFormatSync {
@@ -64,13 +64,13 @@ public class TableFormatSync {
Map<String, SyncResult> results = new HashMap<>();
for (TargetClient targetClient : targetClients) {
try {
- OneTable oneTable = snapshot.getTable();
+ InternalTable internalTable = snapshot.getTable();
results.put(
targetClient.getTableFormat(),
getSyncResult(
targetClient,
SyncMode.FULL,
- oneTable,
+ internalTable,
client -> client.syncFilesForSnapshot(snapshot.getPartitionedDataFiles()),
startTime,
snapshot.getPendingCommits()));
@@ -146,7 +146,7 @@ public class TableFormatSync {
private SyncResult getSyncResult(
TargetClient client,
SyncMode mode,
- OneTable tableState,
+ InternalTable tableState,
SyncFiles fileSyncMethod,
Instant startTime,
List<Instant> pendingCommits) {
diff --git a/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java b/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
index e04af38e..bf2c3809 100644
--- a/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
+++ b/api/src/main/java/org/apache/xtable/spi/sync/TargetClient.java
@@ -24,12 +24,12 @@ import java.util.Optional;
import org.apache.hadoop.conf.Configuration;
import org.apache.xtable.client.PerTableConfig;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
/** A client that provides the major functionality for syncing changes to a target system. */
public interface TargetClient {
@@ -62,7 +62,7 @@ public interface TargetClient {
*
* @param partitionedDataFiles the files to sync, grouped by partition
*/
- void syncFilesForSnapshot(List<OneFileGroup> partitionedDataFiles);
+ void syncFilesForSnapshot(List<PartitionFileGroup> partitionedDataFiles);
/**
* Syncs the changes in files to the target system. This method is required to both add and remove
@@ -77,7 +77,7 @@ public interface TargetClient {
*
* @param table the table that will be synced
*/
- void beginSync(OneTable table);
+ void beginSync(InternalTable table);
/** Completes the sync and performs any cleanup required. */
void completeSync();
diff --git a/api/src/test/java/org/apache/xtable/model/storage/TestFilesDiff.java b/api/src/test/java/org/apache/xtable/model/storage/TestFilesDiff.java
index c52f7e03..32e33988 100644
--- a/api/src/test/java/org/apache/xtable/model/storage/TestFilesDiff.java
+++ b/api/src/test/java/org/apache/xtable/model/storage/TestFilesDiff.java
@@ -38,8 +38,9 @@ public class TestFilesDiff {
InternalDataFile file1Group2 = InternalDataFile.builder().physicalPath("file1Group2").build();
InternalDataFile file2Group2 = InternalDataFile.builder().physicalPath("file2Group2").build();
- List<OneFileGroup> latestFileGroups =
- OneFileGroup.fromFiles(Arrays.asList(file1Group1, file2Group1, file1Group2, file2Group2));
+ List<PartitionFileGroup> latestFileGroups =
+ PartitionFileGroup.fromFiles(
+ Arrays.asList(file1Group1, file2Group1, file1Group2, file2Group2));
Map<String, File> previousFiles = new HashMap<>();
File file1 = mock(File.class);
diff --git a/api/src/test/java/org/apache/xtable/spi/extractor/TestExtractFromSource.java b/api/src/test/java/org/apache/xtable/spi/extractor/TestExtractFromSource.java
index d5720f9d..de77fad7 100644
--- a/api/src/test/java/org/apache/xtable/spi/extractor/TestExtractFromSource.java
+++ b/api/src/test/java/org/apache/xtable/spi/extractor/TestExtractFromSource.java
@@ -39,21 +39,21 @@ import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.IncrementalTableChanges;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
public class TestExtractFromSource {
private final SourceClient<TestCommit> mockSourceClient = mock(SourceClient.class);
@Test
public void extractSnapshot() {
- OneTable table = OneTable.builder().latestCommitTime(Instant.now()).build();
+ InternalTable table = InternalTable.builder().latestCommitTime(Instant.now()).build();
SchemaCatalog schemaCatalog = new SchemaCatalog(Collections.emptyMap());
- List<OneFileGroup> dataFiles = Collections.emptyList();
+ List<PartitionFileGroup> dataFiles = Collections.emptyList();
InternalSnapshot internalSnapshot =
InternalSnapshot.builder()
.schemaCatalog(schemaCatalog)
@@ -85,8 +85,8 @@ public class TestExtractFromSource {
// drop a file and add a file
InternalDataFile newFile1 = getDataFile("file4.parquet");
- OneTable tableAtFirstInstant =
- OneTable.builder().latestCommitTime(Instant.now().minus(1, ChronoUnit.DAYS)).build();
+ InternalTable tableAtFirstInstant =
+ InternalTable.builder().latestCommitTime(Instant.now().minus(1, ChronoUnit.DAYS)).build();
TableChange tableChangeToReturnAtFirstInstant =
TableChange.builder()
.tableAsOfChange(tableAtFirstInstant)
@@ -106,7 +106,8 @@ public class TestExtractFromSource {
InternalDataFile newFile2 = getDataFile("file5.parquet");
InternalDataFile newFile3 = getDataFile("file6.parquet");
- OneTable tableAtSecondInstant = OneTable.builder().latestCommitTime(Instant.now()).build();
+ InternalTable tableAtSecondInstant =
+ InternalTable.builder().latestCommitTime(Instant.now()).build();
TableChange tableChangeToReturnAtSecondInstant =
TableChange.builder()
.tableAsOfChange(tableAtSecondInstant)
diff --git a/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java b/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
index f48fc8fe..925e71c6 100644
--- a/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
+++ b/api/src/test/java/org/apache/xtable/spi/sync/TestTableFormatSync.java
@@ -41,7 +41,7 @@ import org.junit.jupiter.api.Test;
import org.apache.xtable.model.IncrementalTableChanges;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalField;
@@ -50,7 +50,7 @@ import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.model.sync.SyncMode;
import org.apache.xtable.model.sync.SyncResult;
@@ -62,10 +62,10 @@ public class TestTableFormatSync {
@Test
void syncSnapshotWithFailureForOneFormat() {
Instant start = Instant.now();
- OneTable startingTableState = getTableState(1);
- List<OneFileGroup> fileGroups =
+ InternalTable startingTableState = getTableState(1);
+ List<PartitionFileGroup> fileGroups =
Collections.singletonList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(
Collections.singletonList(
InternalDataFile.builder().physicalPath("/tmp/path/file.parquet").build()))
@@ -118,15 +118,15 @@ public class TestTableFormatSync {
@Test
void syncChangesWithFailureForOneFormat() {
Instant start = Instant.now();
- OneTable tableState1 = getTableState(1);
+ InternalTable tableState1 = getTableState(1);
DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
TableChange tableChange1 =
TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
- OneTable tableState2 = getTableState(2);
+ InternalTable tableState2 = getTableState(2);
DataFilesDiff dataFilesDiff2 = getFilesDiff(2);
TableChange tableChange2 =
TableChange.builder().tableAsOfChange(tableState2).filesDiff(dataFilesDiff2).build();
- OneTable tableState3 = getTableState(3);
+ InternalTable tableState3 = getTableState(3);
DataFilesDiff dataFilesDiff3 = getFilesDiff(3);
TableChange tableChange3 =
TableChange.builder().tableAsOfChange(tableState3).filesDiff(dataFilesDiff3).build();
@@ -205,15 +205,15 @@ public class TestTableFormatSync {
@Test
void syncChangesWithDifferentFormatsAndMetadata() {
Instant start = Instant.now();
- OneTable tableState1 = getTableState(1);
+ InternalTable tableState1 = getTableState(1);
DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
TableChange tableChange1 =
TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
- OneTable tableState2 = getTableState(2);
+ InternalTable tableState2 = getTableState(2);
DataFilesDiff dataFilesDiff2 = getFilesDiff(2);
TableChange tableChange2 =
TableChange.builder().tableAsOfChange(tableState2).filesDiff(dataFilesDiff2).build();
- OneTable tableState3 = getTableState(3);
+ InternalTable tableState3 = getTableState(3);
DataFilesDiff dataFilesDiff3 = getFilesDiff(3);
TableChange tableChange3 =
TableChange.builder().tableAsOfChange(tableState3).filesDiff(dataFilesDiff3).build();
@@ -291,7 +291,7 @@ public class TestTableFormatSync {
@Test
void syncChangesOneFormatWithNoRequiredChanges() {
Instant start = Instant.now();
- OneTable tableState1 = getTableState(1);
+ InternalTable tableState1 = getTableState(1);
DataFilesDiff dataFilesDiff1 = getFilesDiff(1);
TableChange tableChange1 =
TableChange.builder().tableAsOfChange(tableState1).filesDiff(dataFilesDiff1).build();
@@ -342,8 +342,8 @@ public class TestTableFormatSync {
* @param id id to use in the names of certain fields
* @return the table
*/
- private static OneTable getTableState(int id) {
- return OneTable.builder()
+ private static InternalTable getTableState(int id) {
+ return InternalTable.builder()
.tableFormat(TableFormat.HUDI)
.readSchema(InternalSchema.builder().name(String.format("test_schema_%d", id)).build())
.partitioningFields(
@@ -371,7 +371,9 @@ public class TestTableFormatSync {
}
private void verifyBaseClientCalls(
- TargetClient mockClient, OneTable startingTableState, List<Instant> pendingCommitInstants) {
+ TargetClient mockClient,
+ InternalTable startingTableState,
+ List<Instant> pendingCommitInstants) {
verify(mockClient).beginSync(startingTableState);
verify(mockClient).syncSchema(startingTableState.getReadSchema());
verify(mockClient).syncPartitionSpec(startingTableState.getPartitioningFields());
diff --git a/core/src/main/java/org/apache/xtable/client/OneTableClient.java b/core/src/main/java/org/apache/xtable/client/OneTableClient.java
index a4c576dc..6b8f45a0 100644
--- a/core/src/main/java/org/apache/xtable/client/OneTableClient.java
+++ b/core/src/main/java/org/apache/xtable/client/OneTableClient.java
@@ -231,7 +231,7 @@ public class OneTableClient {
.min(Instant::compareTo))
.orElseGet(() -> pendingInstantsStream.min(Instant::compareTo));
if (!earliestInstant.isPresent()) {
- log.info("No previous OneTable sync for target. Falling back to snapshot sync.");
+ log.info("No previous InternalTable sync for target. Falling back to snapshot sync.");
return false;
}
boolean isIncrementalSafeFromInstant =
diff --git a/core/src/main/java/org/apache/xtable/constants/OneTableConstants.java b/core/src/main/java/org/apache/xtable/constants/OneTableConstants.java
index aeb43202..1f763dc3 100644
--- a/core/src/main/java/org/apache/xtable/constants/OneTableConstants.java
+++ b/core/src/main/java/org/apache/xtable/constants/OneTableConstants.java
@@ -30,6 +30,6 @@ public class OneTableConstants {
*/
public static final Integer NUM_ARCHIVED_SYNCS_RESULTS = 10;
- /** OneTable meta directory inside table base path to store sync info. */
+ /** InternalTable meta directory inside table base path to store sync info. */
public static final String ONETABLE_META_DIR = ".onetable";
}
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaClient.java b/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
index 906a4c5f..25b26e34 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaClient.java
@@ -56,12 +56,12 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.xtable.client.PerTableConfig;
import org.apache.xtable.exception.NotSupportedException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.spi.sync.TargetClient;
@@ -148,7 +148,7 @@ public class DeltaClient implements TargetClient {
}
@Override
- public void beginSync(OneTable table) {
+ public void beginSync(InternalTable table) {
this.transactionState =
new TransactionState(deltaLog, tableName, table.getLatestCommitTime(), logRetentionInHours);
}
@@ -179,7 +179,7 @@ public class DeltaClient implements TargetClient {
}
@Override
- public void syncFilesForSnapshot(List<OneFileGroup> partitionedDataFiles) {
+ public void syncFilesForSnapshot(List<PartitionFileGroup> partitionedDataFiles) {
transactionState.setActions(
dataFileUpdatesExtractor.applySnapshot(
deltaLog, partitionedDataFiles, transactionState.getLatestSchemaInternal()));
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
index f2ad006f..96776750 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaDataFileUpdatesExtractor.java
@@ -42,7 +42,7 @@ import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.paths.PathUtils;
@Builder
@@ -59,7 +59,9 @@ public class DeltaDataFileUpdatesExtractor {
DeltaDataFileExtractor.builder().build();
public Seq<Action> applySnapshot(
- DeltaLog deltaLog, List<OneFileGroup> partitionedDataFiles, InternalSchema tableSchema) {
+ DeltaLog deltaLog,
+ List<PartitionFileGroup> partitionedDataFiles,
+ InternalSchema tableSchema) {
// all files in the current delta snapshot are potential candidates for remove actions, i.e. if
// the file is not present in the new snapshot (addedFiles) then the file is considered removed
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
index cb0521a1..26418c2d 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java
@@ -45,7 +45,7 @@ import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.schema.SchemaUtils;
/**
- * Converts between Delta and OneTable schemas. Some items to be aware of:
+ * Converts between Delta and InternalTable schemas. Some items to be aware of:
*
* <ul>
* <li>Delta schemas are represented as Spark StructTypes which do not have enums so the enum
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java b/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
index 97605057..c33db77d 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaSourceClient.java
@@ -48,7 +48,7 @@ import org.apache.xtable.exception.OneIOException;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.SchemaCatalog;
@@ -56,7 +56,7 @@ import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.spi.extractor.DataFileIterator;
import org.apache.xtable.spi.extractor.SourceClient;
@@ -81,12 +81,12 @@ public class DeltaSourceClient implements SourceClient<Long> {
private final String basePath;
@Override
- public OneTable getTable(Long version) {
+ public InternalTable getTable(Long version) {
return tableExtractor.table(deltaLog, tableName, version);
}
@Override
- public SchemaCatalog getSchemaCatalog(OneTable table, Long version) {
+ public SchemaCatalog getSchemaCatalog(InternalTable table, Long version) {
// TODO: Does not support schema versions for now
Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
@@ -98,7 +98,7 @@ public class DeltaSourceClient implements SourceClient<Long> {
public InternalSnapshot getCurrentSnapshot() {
DeltaLog deltaLog = DeltaLog.forTable(sparkSession, basePath);
Snapshot snapshot = deltaLog.snapshot();
- OneTable table = getTable(snapshot.version());
+ InternalTable table = getTable(snapshot.version());
return InternalSnapshot.builder()
.table(table)
.schemaCatalog(getSchemaCatalog(table, snapshot.version()))
@@ -108,7 +108,7 @@ public class DeltaSourceClient implements SourceClient<Long> {
@Override
public TableChange getTableChangeForCommit(Long versionNumber) {
- OneTable tableAtVersion = tableExtractor.table(deltaLog, tableName, versionNumber);
+ InternalTable tableAtVersion = tableExtractor.table(deltaLog, tableName, versionNumber);
// Client to call getCommitsBacklog and call this method.
List<Action> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
Snapshot snapshotAtVersion = deltaLog.getSnapshotAt(versionNumber, Option.empty());
@@ -188,11 +188,11 @@ public class DeltaSourceClient implements SourceClient<Long> {
.build());
}
- private List<OneFileGroup> getInternalDataFiles(Snapshot snapshot, InternalSchema schema) {
+ private List<PartitionFileGroup> getInternalDataFiles(Snapshot snapshot, InternalSchema schema) {
try (DataFileIterator fileIterator = dataFileExtractor.iterator(snapshot, schema)) {
List<InternalDataFile> dataFiles = new ArrayList<>();
fileIterator.forEachRemaining(dataFiles::add);
- return OneFileGroup.fromFiles(dataFiles);
+ return PartitionFileGroup.fromFiles(dataFiles);
} catch (Exception e) {
throw new OneIOException("Failed to iterate through Delta data files", e);
}
diff --git a/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java b/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
index efab3128..4bedf1e4 100644
--- a/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
+++ b/core/src/main/java/org/apache/xtable/delta/DeltaTableExtractor.java
@@ -28,19 +28,21 @@ import org.apache.spark.sql.delta.Snapshot;
import scala.Option;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.TableFormat;
-/** Extracts {@link OneTable} canonical representation of a table at a point in time for Delta. */
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
@Builder
public class DeltaTableExtractor {
@Builder.Default
private static final DeltaSchemaExtractor schemaExtractor = DeltaSchemaExtractor.getInstance();
- public OneTable table(DeltaLog deltaLog, String tableName, Long version) {
+ public InternalTable table(DeltaLog deltaLog, String tableName, Long version) {
Snapshot snapshot = deltaLog.getSnapshotAt(version, Option.empty());
InternalSchema schema = schemaExtractor.toInternalSchema(snapshot.metadata().schema());
List<InternalPartitionField> partitionFields =
@@ -52,7 +54,7 @@ public class DeltaTableExtractor {
!partitionFields.isEmpty()
? DataLayoutStrategy.HIVE_STYLE_PARTITION
: DataLayoutStrategy.FLAT;
- return OneTable.builder()
+ return InternalTable.builder()
.tableFormat(TableFormat.DELTA)
.basePath(snapshot.deltaLog().dataPath().toString())
.name(tableName)
diff --git a/core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java b/core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java
index 60ef320c..797662ae 100644
--- a/core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java
@@ -56,7 +56,7 @@ import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.model.stat.ColumnStat;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
@AllArgsConstructor(staticName = "of")
public class BaseFileUpdatesExtractor {
@@ -75,7 +75,9 @@ public class BaseFileUpdatesExtractor {
* @return The information needed to create a "replace" commit for the Hudi table
*/
ReplaceMetadata extractSnapshotChanges(
- List<OneFileGroup> partitionedDataFiles, HoodieTableMetaClient metaClient, String commit) {
+ List<PartitionFileGroup> partitionedDataFiles,
+ HoodieTableMetaClient metaClient,
+ String commit) {
HoodieMetadataConfig metadataConfig =
HoodieMetadataConfig.newBuilder()
.enable(metaClient.getTableConfig().isMetadataTableAvailable())
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiClient.java b/core/src/main/java/org/apache/xtable/hudi/HudiClient.java
index c8219398..2cdc4015 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiClient.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiClient.java
@@ -47,7 +47,7 @@ import org.apache.xtable.exception.OneIOException;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.spi.extractor.SourceClient;
@@ -73,12 +73,12 @@ public class HudiClient implements SourceClient<HoodieInstant> {
}
@Override
- public OneTable getTable(HoodieInstant commit) {
+ public InternalTable getTable(HoodieInstant commit) {
return tableExtractor.table(metaClient, commit);
}
@Override
- public SchemaCatalog getSchemaCatalog(OneTable table, HoodieInstant commit) {
+ public SchemaCatalog getSchemaCatalog(InternalTable table, HoodieInstant commit) {
return HudiSchemaCatalogExtractor.catalogWithTableSchema(table);
}
@@ -97,7 +97,7 @@ public class HudiClient implements SourceClient<HoodieInstant> {
.filterInflightsAndRequested()
.findInstantsBefore(latestCommit.getTimestamp())
.getInstants();
- OneTable table = getTable(latestCommit);
+ InternalTable table = getTable(latestCommit);
return InternalSnapshot.builder()
.table(table)
.schemaCatalog(getSchemaCatalog(table, latestCommit))
@@ -118,7 +118,7 @@ public class HudiClient implements SourceClient<HoodieInstant> {
activeTimeline
.filterCompletedInstants()
.findInstantsBeforeOrEquals(hoodieInstantForDiff.getTimestamp());
- OneTable table = getTable(hoodieInstantForDiff);
+ InternalTable table = getTable(hoodieInstantForDiff);
return TableChange.builder()
.tableAsOfChange(table)
.filesDiff(
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiDataFileExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiDataFileExtractor.java
index 05ddecc4..1d1b2377 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiDataFileExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiDataFileExtractor.java
@@ -60,16 +60,16 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.xtable.collectors.CustomCollectors;
import org.apache.xtable.exception.OneIOException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.SchemaVersion;
import org.apache.xtable.model.stat.PartitionValue;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
-/** Extracts all the files for Hudi table represented by {@link OneTable}. */
+/** Extracts all the files for Hudi table represented by {@link InternalTable}. */
public class HudiDataFileExtractor implements AutoCloseable {
private static final SchemaVersion DEFAULT_SCHEMA_VERSION = new SchemaVersion(1, null);
private final HoodieTableMetadata tableMetadata;
@@ -109,7 +109,7 @@ public class HudiDataFileExtractor implements AutoCloseable {
this.fileStatsExtractor = hudiFileStatsExtractor;
}
- public List<OneFileGroup> getFilesCurrentState(OneTable table) {
+ public List<PartitionFileGroup> getFilesCurrentState(InternalTable table) {
try {
List<String> allPartitionPaths =
tableMetadata != null
@@ -124,7 +124,7 @@ public class HudiDataFileExtractor implements AutoCloseable {
public DataFilesDiff getDiffForCommit(
HoodieInstant hoodieInstantForDiff,
- OneTable table,
+ InternalTable table,
HoodieInstant instant,
HoodieTimeline visibleTimeline) {
SyncableFileSystemView fsView = fileSystemViewManager.getFileSystemView(metaClient);
@@ -343,8 +343,8 @@ public class HudiDataFileExtractor implements AutoCloseable {
return AddedAndRemovedFiles.builder().added(filesToAdd).removed(filesToRemove).build();
}
- private List<OneFileGroup> getInternalDataFilesForPartitions(
- List<String> partitionPaths, OneTable table) {
+ private List<PartitionFileGroup> getInternalDataFilesForPartitions(
+ List<String> partitionPaths, InternalTable table) {
SyncableFileSystemView fsView = fileSystemViewManager.getFileSystemView(metaClient);
Stream<InternalDataFile> filesWithoutStats =
@@ -361,7 +361,7 @@ public class HudiDataFileExtractor implements AutoCloseable {
});
Stream<InternalDataFile> files =
fileStatsExtractor.addStatsToFiles(tableMetadata, filesWithoutStats, table.getReadSchema());
- return OneFileGroup.fromFiles(files);
+ return PartitionFileGroup.fromFiles(files);
}
@Override
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
index d6d0f0e2..0258621f 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiSchemaCatalogExtractor.java
@@ -23,7 +23,7 @@ import java.util.Map;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.SchemaCatalog;
import org.apache.xtable.model.schema.SchemaVersion;
@@ -37,7 +37,7 @@ public class HudiSchemaCatalogExtractor implements SchemaCatalogExtractor<Hoodie
throw new UnsupportedOperationException("Schema catalog extractor not implemented for Hudi");
}
- public static SchemaCatalog catalogWithTableSchema(OneTable table) {
+ public static SchemaCatalog catalogWithTableSchema(InternalTable table) {
// does not support schema versions for now
Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion = new SchemaVersion(1, "");
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java b/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
index a4255f98..a3380b34 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiTableExtractor.java
@@ -33,7 +33,7 @@ import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.util.Option;
import org.apache.xtable.exception.SchemaExtractorException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -42,7 +42,9 @@ import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.schema.SchemaFieldFinder;
import org.apache.xtable.spi.extractor.SourcePartitionSpecExtractor;
-/** Extracts {@link OneTable} a canonical representation of table at a point in time for Hudi. */
+/**
+ * Extracts {@link InternalTable} a canonical representation of table at a point in time for Hudi.
+ */
@Singleton
public class HudiTableExtractor {
private final HudiSchemaExtractor schemaExtractor;
@@ -55,7 +57,7 @@ public class HudiTableExtractor {
this.partitionSpecExtractor = sourcePartitionSpecExtractor;
}
- public OneTable table(HoodieTableMetaClient metaClient, HoodieInstant commit) {
+ public InternalTable table(HoodieTableMetaClient metaClient, HoodieInstant commit) {
TableSchemaResolver tableSchemaResolver = new TableSchemaResolver(metaClient);
InternalSchema canonicalSchema;
Schema avroSchema;
@@ -77,7 +79,7 @@ public class HudiTableExtractor {
partitionFields.size() > 0
? DataLayoutStrategy.DIR_HIERARCHY_PARTITION_VALUES
: DataLayoutStrategy.FLAT;
- return OneTable.builder()
+ return InternalTable.builder()
.tableFormat(TableFormat.HUDI)
.basePath(metaClient.getBasePathV2().toString())
.name(metaClient.getTableConfig().getTableName())
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiTableManager.java b/core/src/main/java/org/apache/xtable/hudi/HudiTableManager.java
index 274387da..1c3d237d 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiTableManager.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiTableManager.java
@@ -36,7 +36,7 @@ import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.exception.TableNotFoundException;
import org.apache.xtable.exception.OneIOException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.storage.DataLayoutStrategy;
@@ -76,13 +76,13 @@ class HudiTableManager {
}
/**
- * Initializes a Hudi table with properties matching the provided {@link OneTable}
+ * Initializes a Hudi table with properties matching the provided {@link InternalTable}
*
* @param tableDataPath the base path for the data files in the table
* @param table the table to initialize
* @return {@link HoodieTableMetaClient} for the table that was created
*/
- HoodieTableMetaClient initializeHudiTable(String tableDataPath, OneTable table) {
+ HoodieTableMetaClient initializeHudiTable(String tableDataPath, InternalTable table) {
String recordKeyField = "";
if (table.getReadSchema() != null) {
List<String> recordKeys =
diff --git a/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java b/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
index 8cd62634..baec9d08 100644
--- a/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
+++ b/core/src/main/java/org/apache/xtable/hudi/HudiTargetClient.java
@@ -82,13 +82,13 @@ import org.apache.xtable.avro.AvroSchemaConverter;
import org.apache.xtable.client.PerTableConfig;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.OneIOException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.spi.sync.TargetClient;
@@ -245,7 +245,7 @@ public class HudiTargetClient implements TargetClient {
}
@Override
- public void syncFilesForSnapshot(List<OneFileGroup> partitionedDataFiles) {
+ public void syncFilesForSnapshot(List<PartitionFileGroup> partitionedDataFiles) {
BaseFileUpdatesExtractor.ReplaceMetadata replaceMetadata =
baseFileUpdatesExtractor.extractSnapshotChanges(
partitionedDataFiles, getMetaClient(), commitState.getInstantTime());
@@ -260,7 +260,7 @@ public class HudiTargetClient implements TargetClient {
}
@Override
- public void beginSync(OneTable table) {
+ public void beginSync(InternalTable table) {
if (!metaClient.isPresent()) {
metaClient = Optional.of(hudiTableManager.initializeHudiTable(tableDataPath, table));
} else {
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
index 8c892cb8..a5ccc092 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergClient.java
@@ -44,12 +44,12 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.xtable.client.PerTableConfig;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.storage.DataFilesDiff;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.spi.sync.TargetClient;
@@ -69,7 +69,7 @@ public class IcebergClient implements TargetClient {
private int snapshotRetentionInHours;
private Transaction transaction;
private Table table;
- private OneTable internalTableState;
+ private InternalTable internalTableState;
public IcebergClient() {}
@@ -143,23 +143,23 @@ public class IcebergClient implements TargetClient {
}
@Override
- public void beginSync(OneTable oneTable) {
- initializeTableIfRequired(oneTable);
+ public void beginSync(InternalTable internalTable) {
+ initializeTableIfRequired(internalTable);
transaction = table.newTransaction();
- internalTableState = oneTable;
+ internalTableState = internalTable;
}
- private void initializeTableIfRequired(OneTable oneTable) {
+ private void initializeTableIfRequired(InternalTable internalTable) {
if (table == null) {
table =
tableManager.getOrCreateTable(
catalogConfig,
tableIdentifier,
basePath,
- schemaExtractor.toIceberg(oneTable.getReadSchema()),
+ schemaExtractor.toIceberg(internalTable.getReadSchema()),
partitionSpecExtractor.toIceberg(
- oneTable.getPartitioningFields(),
- schemaExtractor.toIceberg(oneTable.getReadSchema())));
+ internalTable.getPartitioningFields(),
+ schemaExtractor.toIceberg(internalTable.getReadSchema())));
}
}
@@ -199,7 +199,7 @@ public class IcebergClient implements TargetClient {
}
@Override
- public void syncFilesForSnapshot(List<OneFileGroup> partitionedDataFiles) {
+ public void syncFilesForSnapshot(List<PartitionFileGroup> partitionedDataFiles) {
dataFileUpdatesExtractor.applySnapshot(
table,
internalTableState,
@@ -269,7 +269,8 @@ public class IcebergClient implements TargetClient {
currentSnapshot.snapshotId(),
currentSnapshot.parentId());
// if we need to rollback, we must also clear the last sync state since that sync is no
- // longer considered valid. This will force OneTable to fall back to a snapshot sync in
+ // longer considered valid. This will force InternalTable to fall back to a snapshot sync
+ // in
// the subsequent sync round.
table.manageSnapshots().rollbackTo(currentSnapshot.parentId()).commit();
Transaction transaction = table.newTransaction();
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergColumnStatsConverter.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergColumnStatsConverter.java
index 267d306c..539d2c69 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergColumnStatsConverter.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergColumnStatsConverter.java
@@ -54,7 +54,7 @@ public class IcebergColumnStatsConverter {
Map<Integer, Long> columnSizes = new HashMap<>();
Map<Integer, Long> valueCounts = new HashMap<>();
Map<Integer, Long> nullValueCounts = new HashMap<>();
- Map<Integer, Long> nanValueCounts = null; // OneTable currently doesn't track this
+ Map<Integer, Long> nanValueCounts = null; // InternalTable currently doesn't track this
Map<Integer, ByteBuffer> lowerBounds = new HashMap<>();
Map<Integer, ByteBuffer> upperBounds = new HashMap<>();
fieldColumnStats.forEach(
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
index 18803e57..1b9d70f3 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileExtractor.java
@@ -46,7 +46,7 @@ public class IcebergDataFileExtractor {
* @param dataFile Iceberg data file
* @param partitionValues representation of partition fields and ranges
* @param schema current schema for the table, used for mapping field IDs to stats
- * @return corresponding OneTable data file
+ * @return corresponding InternalTable data file
*/
InternalDataFile fromIceberg(
DataFile dataFile, List<PartitionValue> partitionValues, InternalSchema schema) {
@@ -85,10 +85,10 @@ public class IcebergDataFileExtractor {
}
/**
- * Maps Iceberg file format to OneTable file format
+ * Maps Iceberg file format to InternalTable file format
*
* @param format Iceberg file format
- * @return corresponding OneTable file format
+ * @return corresponding InternalTable file format
*/
FileFormat fromIcebergFileFormat(org.apache.iceberg.FileFormat format) {
switch (format) {
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileUpdatesSync.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileUpdatesSync.java
index 25bfb185..9fd60bf7 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileUpdatesSync.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergDataFileUpdatesSync.java
@@ -29,11 +29,11 @@ import org.apache.iceberg.io.CloseableIterable;
import org.apache.xtable.exception.NotSupportedException;
import org.apache.xtable.exception.OneIOException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FilesDiff;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
@AllArgsConstructor(staticName = "of")
public class IcebergDataFileUpdatesSync {
@@ -42,9 +42,9 @@ public class IcebergDataFileUpdatesSync {
public void applySnapshot(
Table table,
- OneTable oneTable,
+ InternalTable internalTable,
Transaction transaction,
- List<OneFileGroup> partitionedDataFiles,
+ List<PartitionFileGroup> partitionedDataFiles,
Schema schema,
PartitionSpec partitionSpec) {
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionValueConverter.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionValueConverter.java
index 73088d52..bbda45a7 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionValueConverter.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergPartitionValueConverter.java
@@ -44,7 +44,7 @@ import org.apache.iceberg.types.Types;
import org.apache.xtable.avro.AvroSchemaConverter;
import org.apache.xtable.exception.NotSupportedException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.PartitionTransformType;
@@ -72,13 +72,13 @@ public class IcebergPartitionValueConverter {
}
public List<PartitionValue> toOneTable(
- OneTable oneTable, StructLike structLike, PartitionSpec partitionSpec) {
+ InternalTable internalTable, StructLike structLike, PartitionSpec partitionSpec) {
if (!partitionSpec.isPartitioned()) {
return Collections.emptyList();
}
List<PartitionValue> partitionValues = new ArrayList<>(partitionSpec.fields().size());
Map<InternalField, Map<PartitionTransformType, InternalPartitionField>> partitionFieldMap =
- getInternalPartitionFieldMap(oneTable);
+ getInternalPartitionFieldMap(internalTable);
IndexedRecord partitionData = ((IndexedRecord) structLike);
for (PartitionField partitionField : partitionSpec.fields()) {
Object value;
@@ -131,7 +131,7 @@ public class IcebergPartitionValueConverter {
partitionSpec.schema().findField(partitionField.sourceId());
InternalField sourceField =
SchemaFieldFinder.getInstance()
- .findFieldByPath(oneTable.getReadSchema(), partitionSourceField.name());
+ .findFieldByPath(internalTable.getReadSchema(), partitionSourceField.name());
// This helps reduce creating these objects for each file processed and re-using them.
InternalPartitionField internalPartitionField =
getFromInternalPartitionFieldMap(partitionFieldMap, sourceField, transformType);
@@ -163,8 +163,8 @@ public class IcebergPartitionValueConverter {
}
private Map<InternalField, Map<PartitionTransformType, InternalPartitionField>>
- getInternalPartitionFieldMap(OneTable oneTable) {
- List<InternalPartitionField> internalPartitionFields = oneTable.getPartitioningFields();
+ getInternalPartitionFieldMap(InternalTable internalTable) {
+ List<InternalPartitionField> internalPartitionFields = internalTable.getPartitioningFields();
return internalPartitionFields.stream()
.collect(
Collectors.groupingBy(
diff --git a/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java b/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
index 1820573a..415356c7 100644
--- a/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
+++ b/core/src/main/java/org/apache/xtable/iceberg/IcebergSourceClient.java
@@ -41,7 +41,7 @@ import org.apache.xtable.exception.OneIOException;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -91,7 +91,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
}
@Override
- public OneTable getTable(Snapshot snapshot) {
+ public InternalTable getTable(Snapshot snapshot) {
Table iceTable = getSourceTable();
Schema iceSchema = iceTable.schemas().get(snapshot.schemaId());
IcebergSchemaExtractor schemaExtractor = IcebergSchemaExtractor.getInstance();
@@ -107,7 +107,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
irPartitionFields.size() > 0
? DataLayoutStrategy.HIVE_STYLE_PARTITION
: DataLayoutStrategy.FLAT;
- return OneTable.builder()
+ return InternalTable.builder()
.tableFormat(TableFormat.ICEBERG)
.basePath(iceTable.location())
.name(iceTable.name())
@@ -119,7 +119,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
}
@Override
- public SchemaCatalog getSchemaCatalog(OneTable table, Snapshot snapshot) {
+ public SchemaCatalog getSchemaCatalog(InternalTable table, Snapshot snapshot) {
Table iceTable = getSourceTable();
Integer iceSchemaId = snapshot.schemaId();
Schema iceSchema = iceTable.schemas().get(iceSchemaId);
@@ -135,12 +135,12 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
Table iceTable = getSourceTable();
Snapshot currentSnapshot = iceTable.currentSnapshot();
- OneTable irTable = getTable(currentSnapshot);
+ InternalTable irTable = getTable(currentSnapshot);
SchemaCatalog schemaCatalog = getSchemaCatalog(irTable, currentSnapshot);
TableScan scan = iceTable.newScan().useSnapshot(currentSnapshot.snapshotId());
PartitionSpec partitionSpec = iceTable.spec();
- List<OneFileGroup> partitionedDataFiles;
+ List<PartitionFileGroup> partitionedDataFiles;
try (CloseableIterable<FileScanTask> files = scan.planFiles()) {
List<InternalDataFile> irFiles = new ArrayList<>();
for (FileScanTask fileScanTask : files) {
@@ -148,7 +148,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
InternalDataFile irDataFile = fromIceberg(file, partitionSpec, irTable);
irFiles.add(irDataFile);
}
- partitionedDataFiles = OneFileGroup.fromFiles(irFiles);
+ partitionedDataFiles = PartitionFileGroup.fromFiles(irFiles);
} catch (IOException e) {
throw new OneIOException("Failed to fetch current snapshot files from Iceberg source", e);
}
@@ -162,10 +162,10 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
}
private InternalDataFile fromIceberg(
- DataFile file, PartitionSpec partitionSpec, OneTable oneTable) {
+ DataFile file, PartitionSpec partitionSpec, InternalTable internalTable) {
List<PartitionValue> partitionValues =
- partitionConverter.toOneTable(oneTable, file.partition(), partitionSpec);
- return dataFileExtractor.fromIceberg(file, partitionValues, oneTable.getReadSchema());
+ partitionConverter.toOneTable(internalTable, file.partition(), partitionSpec);
+ return dataFileExtractor.fromIceberg(file, partitionValues, internalTable.getReadSchema());
}
@Override
@@ -173,7 +173,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
FileIO fileIO = getTableOps();
Table iceTable = getSourceTable();
PartitionSpec partitionSpec = iceTable.spec();
- OneTable irTable = getTable(snapshot);
+ InternalTable irTable = getTable(snapshot);
Set<InternalDataFile> dataFilesAdded =
StreamSupport.stream(snapshot.addedDataFiles(fileIO).spliterator(), false)
@@ -188,7 +188,7 @@ public class IcebergSourceClient implements SourceClient<Snapshot> {
DataFilesDiff filesDiff =
DataFilesDiff.builder().filesAdded(dataFilesAdded).filesRemoved(dataFilesRemoved).build();
- OneTable table = getTable(snapshot);
+ InternalTable table = getTable(snapshot);
return TableChange.builder().tableAsOfChange(table).filesDiff(filesDiff).build();
}
diff --git a/core/src/test/java/org/apache/xtable/ValidationTestHelper.java b/core/src/test/java/org/apache/xtable/ValidationTestHelper.java
index a44c4ea0..a20ae4e7 100644
--- a/core/src/test/java/org/apache/xtable/ValidationTestHelper.java
+++ b/core/src/test/java/org/apache/xtable/ValidationTestHelper.java
@@ -37,16 +37,16 @@ public class ValidationTestHelper {
InternalSnapshot internalSnapshot, List<String> allActivePaths) {
assertNotNull(internalSnapshot);
assertNotNull(internalSnapshot.getTable());
- List<String> onetablePaths =
+ List<String> filePaths =
internalSnapshot.getPartitionedDataFiles().stream()
.flatMap(group -> group.getFiles().stream())
.map(InternalDataFile::getPhysicalPath)
.collect(Collectors.toList());
replaceFileScheme(allActivePaths);
- replaceFileScheme(onetablePaths);
+ replaceFileScheme(filePaths);
Collections.sort(allActivePaths);
- Collections.sort(onetablePaths);
- assertEquals(allActivePaths, onetablePaths);
+ Collections.sort(filePaths);
+ assertEquals(allActivePaths, filePaths);
}
public static void validateTableChanges(
@@ -90,7 +90,7 @@ public class ValidationTestHelper {
public static List<String> getAllFilePaths(InternalSnapshot internalSnapshot) {
return internalSnapshot.getPartitionedDataFiles().stream()
- .flatMap(oneFileGroup -> oneFileGroup.getFiles().stream())
+ .flatMap(fileGroup -> fileGroup.getFiles().stream())
.map(InternalDataFile::getPhysicalPath)
.collect(Collectors.toList());
}
diff --git a/core/src/test/java/org/apache/xtable/client/TestOneTableClient.java b/core/src/test/java/org/apache/xtable/client/TestOneTableClient.java
index adbeba3f..fe8afa09 100644
--- a/core/src/test/java/org/apache/xtable/client/TestOneTableClient.java
+++ b/core/src/test/java/org/apache/xtable/client/TestOneTableClient.java
@@ -47,7 +47,7 @@ import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.IncrementalTableChanges;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.storage.TableFormat;
@@ -72,8 +72,8 @@ public class TestOneTableClient {
@Test
void testAllSnapshotSyncAsPerConfig() {
SyncMode syncMode = SyncMode.FULL;
- OneTable oneTable = getOneTable();
- InternalSnapshot internalSnapshot = buildSnapshot(oneTable, "v1");
+ InternalTable internalTable = getOneTable();
+ InternalSnapshot internalSnapshot = buildSnapshot(internalTable, "v1");
Instant instantBeforeHour = Instant.now().minus(Duration.ofHours(1));
SyncResult syncResult = buildSyncResult(syncMode, instantBeforeHour);
Map<String, SyncResult> perTableResults = new HashMap<>();
@@ -184,9 +184,9 @@ public class TestOneTableClient {
@Test
void testIncrementalSyncFallBackToSnapshotForAllFormats() {
SyncMode syncMode = SyncMode.INCREMENTAL;
- OneTable oneTable = getOneTable();
+ InternalTable internalTable = getOneTable();
Instant instantBeforeHour = Instant.now().minus(Duration.ofHours(1));
- InternalSnapshot internalSnapshot = buildSnapshot(oneTable, "v1");
+ InternalSnapshot internalSnapshot = buildSnapshot(internalTable, "v1");
SyncResult syncResult = buildSyncResult(syncMode, instantBeforeHour);
Map<String, SyncResult> syncResults = new HashMap<>();
syncResults.put(TableFormat.ICEBERG, syncResult);
@@ -273,9 +273,9 @@ public class TestOneTableClient {
when(mockSourceClient.getTableChangeForCommit(instant)).thenReturn(tableChange);
}
// Iceberg needs to sync by snapshot since instant15 is affected by table clean-up.
- OneTable oneTable = getOneTable();
+ InternalTable internalTable = getOneTable();
Instant instantBeforeHour = Instant.now().minus(Duration.ofHours(1));
- InternalSnapshot internalSnapshot = buildSnapshot(oneTable, "v1");
+ InternalSnapshot internalSnapshot = buildSnapshot(internalTable, "v1");
SyncResult syncResult = buildSyncResult(syncMode, instantBeforeHour);
Map<String, SyncResult> snapshotResult =
Collections.singletonMap(TableFormat.ICEBERG, syncResult);
@@ -378,16 +378,16 @@ public class TestOneTableClient {
.build();
}
- private InternalSnapshot buildSnapshot(OneTable oneTable, String version) {
- return InternalSnapshot.builder().table(oneTable).version(version).build();
+ private InternalSnapshot buildSnapshot(InternalTable internalTable, String version) {
+ return InternalSnapshot.builder().table(internalTable).version(version).build();
}
- private OneTable getOneTable() {
+ private InternalTable getOneTable() {
return getOneTable(Instant.now());
}
- private OneTable getOneTable(Instant instant) {
- return OneTable.builder().name("some_table").latestCommitTime(instant).build();
+ private InternalTable getOneTable(Instant instant) {
+ return InternalTable.builder().name("some_table").latestCommitTime(instant).build();
}
private Instant getInstantAtLastNMinutes(Instant currentInstant, int n) {
diff --git a/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java b/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
index 52ac9ed0..3a0f589a 100644
--- a/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
+++ b/core/src/test/java/org/apache/xtable/delta/ITDeltaSourceClient.java
@@ -58,7 +58,7 @@ import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
@@ -199,7 +199,7 @@ public class ITDeltaSourceClient {
List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
validatePartitionDataFiles(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(
Collections.singletonList(
InternalDataFile.builder()
@@ -286,7 +286,7 @@ public class ITDeltaSourceClient {
.range(Range.scalar("SingleValue"))
.build());
validatePartitionDataFiles(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.partitionValues(partitionValue)
.files(
Collections.singletonList(
@@ -665,19 +665,19 @@ public class ITDeltaSourceClient {
}
private static void validateTable(
- OneTable oneTable,
+ InternalTable internalTable,
String tableName,
String tableFormat,
InternalSchema readSchema,
DataLayoutStrategy dataLayoutStrategy,
String basePath,
List<InternalPartitionField> partitioningFields) {
- Assertions.assertEquals(tableName, oneTable.getName());
- Assertions.assertEquals(tableFormat, oneTable.getTableFormat());
- Assertions.assertEquals(readSchema, oneTable.getReadSchema());
- Assertions.assertEquals(dataLayoutStrategy, oneTable.getLayoutStrategy());
- Assertions.assertEquals(basePath, oneTable.getBasePath());
- Assertions.assertEquals(partitioningFields, oneTable.getPartitioningFields());
+ Assertions.assertEquals(tableName, internalTable.getName());
+ Assertions.assertEquals(tableFormat, internalTable.getTableFormat());
+ Assertions.assertEquals(readSchema, internalTable.getReadSchema());
+ Assertions.assertEquals(dataLayoutStrategy, internalTable.getLayoutStrategy());
+ Assertions.assertEquals(basePath, internalTable.getBasePath());
+ Assertions.assertEquals(partitioningFields, internalTable.getPartitioningFields());
}
private void validateSchemaCatalog(
@@ -686,7 +686,7 @@ public class ITDeltaSourceClient {
}
private void validatePartitionDataFiles(
- OneFileGroup expectedPartitionFiles, OneFileGroup actualPartitionFiles)
+ PartitionFileGroup expectedPartitionFiles, PartitionFileGroup actualPartitionFiles)
throws URISyntaxException {
assertEquals(
expectedPartitionFiles.getPartitionValues(), actualPartitionFiles.getPartitionValues());
diff --git a/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java b/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
index 936dddb8..b248a630 100644
--- a/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
+++ b/core/src/test/java/org/apache/xtable/delta/TestDeltaSync.java
@@ -78,7 +78,7 @@ import io.delta.standalone.types.StringType;
import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -89,7 +89,7 @@ import org.apache.xtable.model.stat.Range;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.schema.SchemaFieldFinder;
import org.apache.xtable.spi.sync.TableFormatSync;
@@ -150,8 +150,8 @@ public class TestDeltaSync {
.build())
.build());
InternalSchema schema2 = getInternalSchema().toBuilder().fields(fields2).build();
- OneTable table1 = getOneTable(tableName, basePath, schema1, null, LAST_COMMIT_TIME);
- OneTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
+ InternalTable table1 = getOneTable(tableName, basePath, schema1, null, LAST_COMMIT_TIME);
+ InternalTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
InternalDataFile dataFile1 = getDataFile(1, Collections.emptyList(), basePath);
InternalDataFile dataFile2 = getDataFile(2, Collections.emptyList(), basePath);
@@ -183,7 +183,7 @@ public class TestDeltaSync {
.build())
.transformType(PartitionTransformType.VALUE)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -241,7 +241,7 @@ public class TestDeltaSync {
.build())
.transformType(PartitionTransformType.VALUE)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -307,7 +307,7 @@ public class TestDeltaSync {
.sourceField(SchemaFieldFinder.getInstance().findFieldByPath(schema, "timestamp_field"))
.transformType(transformType)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -401,20 +401,20 @@ public class TestDeltaSync {
internalDataFiles.size(), count, "Number of files from DeltaScan don't match expectation");
}
- private InternalSnapshot buildSnapshot(OneTable table, InternalDataFile... dataFiles) {
+ private InternalSnapshot buildSnapshot(InternalTable table, InternalDataFile... dataFiles) {
return InternalSnapshot.builder()
.table(table)
- .partitionedDataFiles(OneFileGroup.fromFiles(Arrays.asList(dataFiles)))
+ .partitionedDataFiles(PartitionFileGroup.fromFiles(Arrays.asList(dataFiles)))
.build();
}
- private OneTable getOneTable(
+ private InternalTable getOneTable(
String tableName,
Path basePath,
InternalSchema schema,
List<InternalPartitionField> partitionFields,
Instant lastCommitTime) {
- return OneTable.builder()
+ return InternalTable.builder()
.name(tableName)
.basePath(basePath.toUri().toString())
.layoutStrategy(DataLayoutStrategy.FLAT)
diff --git a/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java b/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
index 74db2ed9..3c81ac55 100644
--- a/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
+++ b/core/src/test/java/org/apache/xtable/hudi/ITHudiTargetClient.java
@@ -71,7 +71,7 @@ import org.apache.hudi.metadata.HoodieBackedTableMetadata;
import org.apache.hudi.metadata.HoodieMetadataFileSystemView;
import org.apache.xtable.client.PerTableConfigImpl;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
@@ -85,7 +85,7 @@ import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.spi.sync.TargetClient;
@@ -200,7 +200,7 @@ public class ITHudiTargetClient {
.build();
// perform sync
HudiTargetClient targetClient = getTargetClient();
- OneTable initialState = getState(Instant.now());
+ InternalTable initialState = getState(Instant.now());
targetClient.beginSync(initialState);
targetClient.syncFilesForDiff(dataFilesDiff);
targetClient.syncSchema(SCHEMA);
@@ -227,9 +227,9 @@ public class ITHudiTargetClient {
String partitionPath = "partition_path";
String fileName = "file_1.parquet";
String filePath = getFilePath(partitionPath, fileName);
- List<OneFileGroup> snapshot =
+ List<PartitionFileGroup> snapshot =
Collections.singletonList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(Collections.singletonList(getTestFile(partitionPath, fileName)))
.partitionValues(
Collections.singletonList(
@@ -239,7 +239,7 @@ public class ITHudiTargetClient {
.build()))
.build());
// sync snapshot and metadata
- OneTable initialState = getState(Instant.now());
+ InternalTable initialState = getState(Instant.now());
HudiTargetClient targetClient = getTargetClient();
targetClient.beginSync(initialState);
targetClient.syncFilesForSnapshot(snapshot);
@@ -269,9 +269,9 @@ public class ITHudiTargetClient {
String fileName1 = "file_1.parquet";
String filePath1 = getFilePath(partitionPath, fileName1);
- List<OneFileGroup> snapshot =
+ List<PartitionFileGroup> snapshot =
Collections.singletonList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(
Arrays.asList(
getTestFile(partitionPath, fileName0),
@@ -284,7 +284,7 @@ public class ITHudiTargetClient {
.build()))
.build());
// sync snapshot and metadata
- OneTable initialState = getState(Instant.now().minus(24, ChronoUnit.HOURS));
+ InternalTable initialState = getState(Instant.now().minus(24, ChronoUnit.HOURS));
HudiTargetClient targetClient = getTargetClient();
targetClient.beginSync(initialState);
targetClient.syncFilesForSnapshot(snapshot);
@@ -380,7 +380,7 @@ public class ITHudiTargetClient {
Instant commitStart) {
DataFilesDiff dataFilesDiff2 =
DataFilesDiff.builder().filesAdded(filesToAdd).filesRemoved(filesToRemove).build();
- OneTable state3 = getState(commitStart);
+ InternalTable state3 = getState(commitStart);
targetClient.beginSync(state3);
targetClient.syncFilesForDiff(dataFilesDiff2);
OneTableMetadata latestState =
@@ -566,8 +566,8 @@ public class ITHudiTargetClient {
.build();
}
- private OneTable getState(Instant latestCommitTime) {
- return OneTable.builder()
+ private InternalTable getState(Instant latestCommitTime) {
+ return InternalTable.builder()
.basePath(tableBasePath)
.name(TABLE_NAME)
.latestCommitTime(latestCommitTime)
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java b/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
index ccdbda5f..dddc736f 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestBaseFileUpdatesExtractor.java
@@ -63,7 +63,7 @@ import org.apache.xtable.model.stat.Range;
import org.apache.xtable.model.storage.DataFilesDiff;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.testutil.ColumnStatMapUtil;
public class TestBaseFileUpdatesExtractor {
@@ -180,9 +180,9 @@ public class TestBaseFileUpdatesExtractor {
BaseFileUpdatesExtractor extractor =
BaseFileUpdatesExtractor.of(CONTEXT, new CachingPath(tableBasePath));
- List<OneFileGroup> partitionedDataFiles =
+ List<PartitionFileGroup> partitionedDataFiles =
Arrays.asList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.partitionValues(
Collections.singletonList(
PartitionValue.builder()
@@ -191,7 +191,7 @@ public class TestBaseFileUpdatesExtractor {
.build()))
.files(Arrays.asList(addedFile1, addedFile2))
.build(),
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.partitionValues(
Collections.singletonList(
PartitionValue.builder()
@@ -271,9 +271,9 @@ public class TestBaseFileUpdatesExtractor {
createFile(
String.format("%s/%s/%s", tableBasePath, partitionPath2, existingFileName2),
Collections.emptyList());
- List<OneFileGroup> partitionedDataFiles =
+ List<PartitionFileGroup> partitionedDataFiles =
Arrays.asList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(Arrays.asList(addedFile1, existingFile))
.partitionValues(
Collections.singletonList(
@@ -282,7 +282,7 @@ public class TestBaseFileUpdatesExtractor {
.range(Range.scalar(partitionPath2))
.build()))
.build(),
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(Collections.singletonList(addedFile2))
.partitionValues(
Collections.singletonList(
@@ -356,9 +356,9 @@ public class TestBaseFileUpdatesExtractor {
InternalDataFile existingFile =
createFile(
String.format("%s/%s", tableBasePath, existingFileName2), Collections.emptyList());
- List<OneFileGroup> partitionedDataFiles =
+ List<PartitionFileGroup> partitionedDataFiles =
Collections.singletonList(
- OneFileGroup.builder()
+ PartitionFileGroup.builder()
.files(Arrays.asList(addedFile1, existingFile))
.partitionValues(Collections.emptyList())
.build());
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
index 415c8baa..c0d5e6d4 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiTableManager.java
@@ -38,7 +38,7 @@ import org.junit.jupiter.params.provider.MethodSource;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -82,8 +82,8 @@ public class TestHudiTableManager {
.sourceField(InternalField.builder().name(field2).build())
.transformType(PartitionTransformType.VALUE)
.build());
- OneTable table =
- OneTable.builder()
+ InternalTable table =
+ InternalTable.builder()
.name(tableName)
.partitioningFields(inputPartitionFields)
.readSchema(tableSchema)
diff --git a/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java b/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
index ec1abc05..493cb9f0 100644
--- a/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
+++ b/core/src/test/java/org/apache/xtable/hudi/TestHudiTargetClient.java
@@ -42,7 +42,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.xtable.avro.AvroSchemaConverter;
import org.apache.xtable.exception.NotSupportedException;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
@@ -50,7 +50,7 @@ import org.apache.xtable.model.schema.InternalSchema;
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.storage.DataFilesDiff;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
/**
* Unit tests that focus on the basic control flow of the HudiTargetClient. Functional tests are in
@@ -62,8 +62,12 @@ public class TestHudiTargetClient {
private static final Instant COMMIT_TIME = Instant.ofEpochMilli(1598644800000L);
private static final String COMMIT = "20200828200000000";
private static final String BASE_PATH = "test-base-path";
- private static final OneTable TABLE =
- OneTable.builder().name("table").basePath(BASE_PATH).latestCommitTime(COMMIT_TIME).build();
+ private static final InternalTable TABLE =
+ InternalTable.builder()
+ .name("table")
+ .basePath(BASE_PATH)
+ .latestCommitTime(COMMIT_TIME)
+ .build();
private final BaseFileUpdatesExtractor mockBaseFileUpdatesExtractor =
mock(BaseFileUpdatesExtractor.class);
@@ -223,7 +227,7 @@ public class TestHudiTargetClient {
HudiTargetClient.CommitState mockCommitState = mocks.getLeft();
HoodieTableMetaClient mockMetaClient = mocks.getRight();
String instant = "commit";
- List<OneFileGroup> input = Collections.emptyList();
+ List<PartitionFileGroup> input = Collections.emptyList();
BaseFileUpdatesExtractor.ReplaceMetadata output =
BaseFileUpdatesExtractor.ReplaceMetadata.of(
Collections.emptyMap(), Collections.emptyList());
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
index ce141617..52e4f113 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergPartitionValueConverter.java
@@ -32,7 +32,7 @@ import org.apache.iceberg.StructLike;
import org.apache.iceberg.avro.AvroSchemaUtil;
import org.apache.iceberg.types.Types;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -108,13 +108,13 @@ public class TestIcebergPartitionValueConverter {
assertEquals(expectedPartitionValues, partitionValues);
}
- private OneTable buildOnetable(boolean isPartitioned) {
+ private InternalTable buildOnetable(boolean isPartitioned) {
return buildOnetable(isPartitioned, null, null);
}
- private OneTable buildOnetable(
+ private InternalTable buildOnetable(
boolean isPartitioned, String sourceField, PartitionTransformType transformType) {
- return OneTable.builder()
+ return InternalTable.builder()
.readSchema(IcebergSchemaExtractor.getInstance().fromIceberg(SCHEMA))
.partitioningFields(
isPartitioned
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
index 7cead7c6..bca730a8 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSourceClient.java
@@ -54,7 +54,7 @@ import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.CommitsBacklog;
import org.apache.xtable.model.InstantsForIncrementalSync;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.TableChange;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalSchema;
@@ -99,24 +99,25 @@ class TestIcebergSourceClient {
IcebergSourceClient client = clientProvider.getSourceClientInstance(sourceTableConfig);
Snapshot snapshot = catalogSales.currentSnapshot();
- OneTable oneTable = client.getTable(snapshot);
- Assertions.assertNotNull(oneTable);
- assertEquals(TableFormat.ICEBERG, oneTable.getTableFormat());
- Assertions.assertTrue(oneTable.getName().endsWith("catalog_sales"));
- assertEquals(catalogSales.location(), oneTable.getBasePath());
- assertEquals(DataLayoutStrategy.HIVE_STYLE_PARTITION, oneTable.getLayoutStrategy());
- assertEquals(snapshot.timestampMillis(), oneTable.getLatestCommitTime().toEpochMilli());
- Assertions.assertNotNull(oneTable.getReadSchema());
-
- Assertions.assertEquals(7, oneTable.getReadSchema().getFields().size());
- validateSchema(oneTable.getReadSchema(), catalogSales.schema());
-
- assertEquals(1, oneTable.getPartitioningFields().size());
- InternalField partitionField = oneTable.getPartitioningFields().get(0).getSourceField();
+ InternalTable internalTable = client.getTable(snapshot);
+ Assertions.assertNotNull(internalTable);
+ assertEquals(TableFormat.ICEBERG, internalTable.getTableFormat());
+ Assertions.assertTrue(internalTable.getName().endsWith("catalog_sales"));
+ assertEquals(catalogSales.location(), internalTable.getBasePath());
+ assertEquals(DataLayoutStrategy.HIVE_STYLE_PARTITION, internalTable.getLayoutStrategy());
+ assertEquals(snapshot.timestampMillis(), internalTable.getLatestCommitTime().toEpochMilli());
+ Assertions.assertNotNull(internalTable.getReadSchema());
+
+ Assertions.assertEquals(7, internalTable.getReadSchema().getFields().size());
+ validateSchema(internalTable.getReadSchema(), catalogSales.schema());
+
+ assertEquals(1, internalTable.getPartitioningFields().size());
+ InternalField partitionField = internalTable.getPartitioningFields().get(0).getSourceField();
assertEquals("cs_sold_date_sk", partitionField.getName());
assertEquals(7, partitionField.getFieldId());
assertEquals(
- PartitionTransformType.VALUE, oneTable.getPartitioningFields().get(0).getTransformType());
+ PartitionTransformType.VALUE,
+ internalTable.getPartitioningFields().get(0).getTransformType());
}
@Test
@@ -174,9 +175,9 @@ class TestIcebergSourceClient {
verify(spyDataFileExtractor, times(5)).fromIceberg(any(), any(), any());
Assertions.assertNotNull(internalSnapshot.getPartitionedDataFiles());
- List<OneFileGroup> dataFileChunks = internalSnapshot.getPartitionedDataFiles();
+ List<PartitionFileGroup> dataFileChunks = internalSnapshot.getPartitionedDataFiles();
assertEquals(5, dataFileChunks.size());
- for (OneFileGroup dataFilesChunk : dataFileChunks) {
+ for (PartitionFileGroup dataFilesChunk : dataFileChunks) {
List<InternalDataFile> internalDataFiles = dataFilesChunk.getFiles();
assertEquals(1, internalDataFiles.size());
InternalDataFile internalDataFile = internalDataFiles.get(0);
diff --git a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
index 9491ce21..87c244ed 100644
--- a/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
+++ b/core/src/test/java/org/apache/xtable/iceberg/TestIcebergSync.java
@@ -84,7 +84,7 @@ import com.google.common.collect.Sets;
import org.apache.xtable.ITOneTableClient;
import org.apache.xtable.client.PerTableConfigImpl;
import org.apache.xtable.model.InternalSnapshot;
-import org.apache.xtable.model.OneTable;
+import org.apache.xtable.model.InternalTable;
import org.apache.xtable.model.OneTableMetadata;
import org.apache.xtable.model.schema.InternalField;
import org.apache.xtable.model.schema.InternalPartitionField;
@@ -98,7 +98,7 @@ import org.apache.xtable.model.stat.Range;
import org.apache.xtable.model.storage.DataLayoutStrategy;
import org.apache.xtable.model.storage.FileFormat;
import org.apache.xtable.model.storage.InternalDataFile;
-import org.apache.xtable.model.storage.OneFileGroup;
+import org.apache.xtable.model.storage.PartitionFileGroup;
import org.apache.xtable.model.storage.TableFormat;
import org.apache.xtable.schema.SchemaFieldFinder;
import org.apache.xtable.spi.sync.TableFormatSync;
@@ -221,8 +221,8 @@ public class TestIcebergSync {
List<Types.NestedField> fields = new ArrayList<>(icebergSchema.columns());
fields.add(Types.NestedField.of(6, false, "long_field", Types.LongType.get()));
Schema icebergSchema2 = new Schema(fields);
- OneTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
- OneTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
+ InternalTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
+ InternalTable table2 = getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME);
Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion1 = new SchemaVersion(1, "");
schemas.put(schemaVersion1, internalSchema);
@@ -316,8 +316,8 @@ public class TestIcebergSync {
List<Types.NestedField> fields = new ArrayList<>(icebergSchema.columns());
fields.add(Types.NestedField.of(6, false, "long_field", Types.LongType.get()));
Schema icebergSchema2 = new Schema(fields);
- OneTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
- OneTable table2 =
+ InternalTable table1 = getOneTable(tableName, basePath, internalSchema, null, LAST_COMMIT_TIME);
+ InternalTable table2 =
getOneTable(tableName, basePath, schema2, null, LAST_COMMIT_TIME.plusMillis(100000L));
Map<SchemaVersion, InternalSchema> schemas = new HashMap<>();
SchemaVersion schemaVersion1 = new SchemaVersion(1, "");
@@ -379,7 +379,7 @@ public class TestIcebergSync {
.transformType(PartitionTransformType.DAY)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -444,7 +444,7 @@ public class TestIcebergSync {
.transformType(PartitionTransformType.DAY)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -506,7 +506,7 @@ public class TestIcebergSync {
.transformType(PartitionTransformType.VALUE)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -568,7 +568,7 @@ public class TestIcebergSync {
.transformType(PartitionTransformType.DAY)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -645,7 +645,7 @@ public class TestIcebergSync {
.transformType(PartitionTransformType.VALUE)
.build();
- OneTable table =
+ InternalTable table =
getOneTable(
tableName,
basePath,
@@ -696,11 +696,13 @@ public class TestIcebergSync {
}
private InternalSnapshot buildSnapshot(
- OneTable table, Map<SchemaVersion, InternalSchema> schemas, InternalDataFile... dataFiles) {
+ InternalTable table,
+ Map<SchemaVersion, InternalSchema> schemas,
+ InternalDataFile... dataFiles) {
return InternalSnapshot.builder()
.table(table)
.schemaCatalog(SchemaCatalog.builder().schemas(schemas).build())
- .partitionedDataFiles(OneFileGroup.fromFiles(Arrays.asList(dataFiles)))
+ .partitionedDataFiles(PartitionFileGroup.fromFiles(Arrays.asList(dataFiles)))
.build();
}
@@ -718,13 +720,13 @@ public class TestIcebergSync {
.build();
}
- private OneTable getOneTable(
+ private InternalTable getOneTable(
String tableName,
Path basePath,
InternalSchema schema,
List<InternalPartitionField> partitionFields,
Instant lastCommitTime) {
- return OneTable.builder()
+ return InternalTable.builder()
.name(tableName)
.basePath(basePath.toString())
.layoutStrategy(DataLayoutStrategy.FLAT)
@@ -737,7 +739,7 @@ public class TestIcebergSync {
private void validateIcebergTable(
String tableName,
- OneTable table,
+ InternalTable table,
Set<InternalDataFile> expectedFiles,
Expression filterExpression)
throws IOException {
diff --git a/hudi-support/extensions/src/main/java/org/apache/xtable/hudi/sync/OneTableSyncTool.java b/hudi-support/extensions/src/main/java/org/apache/xtable/hudi/sync/OneTableSyncTool.java
index 3d067d1a..b84171b1 100644
--- a/hudi-support/extensions/src/main/java/org/apache/xtable/hudi/sync/OneTableSyncTool.java
+++ b/hudi-support/extensions/src/main/java/org/apache/xtable/hudi/sync/OneTableSyncTool.java
@@ -43,7 +43,10 @@ import org.apache.xtable.model.schema.PartitionTransformType;
import org.apache.xtable.model.sync.SyncMode;
import org.apache.xtable.model.sync.SyncResult;
-/** A HoodieSyncTool for syncing a Hudi table to other formats (Delta and Iceberg) with OneTable. */
+/**
+ * A HoodieSyncTool for syncing a Hudi table to other formats (Delta and Iceberg) with
+ * InternalTable.
+ */
public class OneTableSyncTool extends HoodieSyncTool {
private final OneTableSyncConfig config;
private final HudiSourceClientProvider hudiSourceClientProvider;
@@ -87,7 +90,7 @@ public class OneTableSyncTool extends HoodieSyncTool {
.map(entry -> entry.getKey().toString())
.collect(Collectors.joining(","));
if (!failingFormats.isEmpty()) {
- throw new HoodieException("Unable to sync to OneTable for formats: " + failingFormats);
+ throw new HoodieException("Unable to sync to InternalTable for formats: " + failingFormats);
}
}
diff --git a/hudi-support/extensions/src/test/java/org/apache/xtable/hudi/sync/TestOneTableSyncTool.java b/hudi-support/extensions/src/test/java/org/apache/xtable/hudi/sync/TestOneTableSyncTool.java
index 21922811..59588df4 100644
--- a/hudi-support/extensions/src/test/java/org/apache/xtable/hudi/sync/TestOneTableSyncTool.java
+++ b/hudi-support/extensions/src/test/java/org/apache/xtable/hudi/sync/TestOneTableSyncTool.java
@@ -104,7 +104,8 @@ public class TestOneTableSyncTool {
properties.putAll(options);
new OneTableSyncTool(properties, new Configuration()).syncHoodieTable();
- // lightweight check to make sure metadata dirs are made - assumes that OneTable sync is correct
+ // lightweight check to make sure metadata dirs are made - assumes that InternalTable sync is
+ // correct
// if it succeeds
assertTrue(Files.exists(Paths.get(URI.create(path + "/_delta_log"))));
assertTrue(Files.exists(Paths.get(URI.create(path + "/metadata"))));
diff --git a/utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
index 33dd67c2..9741a0b4 100644
--- a/utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
+++ b/utilities/src/main/java/org/apache/xtable/utilities/RunSync.java
@@ -87,7 +87,7 @@ public class RunSync {
CLIENTS_CONFIG_PATH,
"clientsConfig",
true,
- "The path to a yaml file containing OneTable client configurations. "
+ "The path to a yaml file containing InternalTable client configurations. "
+ "These configs will override the default")
.addOption(
ICEBERG_CATALOG_CONFIG_PATH,
diff --git a/utilities/src/main/resources/onetable-client-defaults.yaml b/utilities/src/main/resources/onetable-client-defaults.yaml
index 90c5b773..93e5cad3 100644
--- a/utilities/src/main/resources/onetable-client-defaults.yaml
+++ b/utilities/src/main/resources/onetable-client-defaults.yaml
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-## This file contains the default configuration of known OneTable tableFormatsClients for both source and target
+## This file contains the default configuration of known InternalTable tableFormatsClients for both source and target
## table formats. The config maps a table format name to a client class, and contains the default configuration
## values needed by the client.