You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@xtable.apache.org by "vaibhavk1992 (via GitHub)" <gi...@apache.org> on 2025/06/30 10:42:21 UTC
[PR] Delta Kernel Draft PR [incubator-xtable]
vaibhavk1992 opened a new pull request, #729:
URL: https://github.com/apache/incubator-xtable/pull/729
## *Important Read*
- *Please ensure the GitHub issue is mentioned at the beginning of the PR*
## What is the purpose of the pull request
*(For example: This pull request implements the sync for delta format.)*
## Brief change log
*(for example:)*
- *Fixed JSON parsing error when persisting state*
- *Added unit tests for schema evolution*
## Verify this pull request
*(Please pick either of the following options)*
This pull request is a trivial rework / code cleanup without any test coverage.
*(or)*
This pull request is already covered by existing tests, such as *(please describe tests)*.
(or)
This change added tests and can be verified as follows:
*(example:)*
- *Added integration tests for end-to-end.*
- *Added TestConversionController to verify the change.*
- *Manually verified the change by running a job locally.*
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559840656
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
Review Comment:
done
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565544894
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Slf4j
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ log.error("Error checking if incremental sync is safe from " + instant + ": " + e);
Review Comment:
```suggestion
log.error("Error checking if incremental sync is safe from " + instant, e);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565567891
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
We'll want these to be the default compile scope. I think that will make it easier than requiring users to provide their own version at runtime.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.exception.SchemaExtractorException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+ List<StructField> partitionFieldSchemas =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFieldSchemas);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
If this is just some general catch-all, my recommendation is to remove it. If this is supposed to catch some specific exception from the schema extraction, let's catch that particular exception.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2538869316
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
it has been added back
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559839838
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
Review Comment:
done.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2567524182
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
I have added this and it passed too. Thanks @the-other-tim-brown
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2406935357
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.spark.sql.types.Metadata;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.delta.ScalaUtils;
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
Review Comment:
Can we have a unit test for this class?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.*;
+
+import io.delta.kernel.types.*;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaPartitionExtractor;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
Review Comment:
Can we have a unit test for this class?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Value;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.internal.actions.AddFile;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaValueConverter;
+import org.apache.xtable.model.exception.ParseException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.stat.Range;
+
+/**
+ * DeltaStatsExtractor extracts column stats and also responsible for their serialization leveraging
+ * {@link DeltaValueConverter}.
+ */
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelStatsExtractor {
Review Comment:
Similarly here, we could use some unit testing.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2410446809
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
Review Comment:
it has been moved to kernel package.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176578472
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.time.Instant;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.delta.DeltaKernelTableExtractor;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.*;
+import org.apache.xtable.spi.extractor.ConversionSource;
+
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
Review Comment:
We will need full implementation of all the interface methods, otherwise this will fail during the table format sync. Can you refer to the impl for DeltaConversionSource for these methods?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176370494
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
This should be added back, any reason why you had to comment this out?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2330613467
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.*;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.*;
+import io.delta.kernel.internal.actions.SingleAction;
+import io.delta.kernel.internal.fs.Path;
+import io.delta.kernel.internal.replay.ActionsIterator;
+import io.delta.kernel.internal.util.FileNames;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ private final StructType actionSchema = SingleAction.FULL_SCHEMA;
+ // private final DeltaKernelTableExtractor tableExtractor;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ Configuration hadoopConf = new Configuration();
+ try {
+ Engine engine = DefaultEngine.create(hadoopConf);
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+ List<FileStatus> files =
+ DeltaLogActionUtils.listDeltaLogFilesAsIter(
+ engine,
+ Collections.singleton(FileNames.DeltaLogFileType.COMMIT),
+ new Path(basePath),
+ versionNumber,
+ Optional.of(versionNumber),
+ false)
+ .toInMemoryList();
+
+ List<Row> actions = new ArrayList<>();
+ ActionsIterator actionsIterator =
+ new ActionsIterator(engine, files, actionSchema, Optional.empty());
+ while (actionsIterator.hasNext()) {
+ // Each ActionWrapper may wrap a batch of rows (actions)
+ CloseableIterator<Row> scanFileRows = actionsIterator.next().getColumnarBatch().getRows();
+ while (scanFileRows.hasNext()) {
+ Row scanFileRow = scanFileRows.next();
+ if (scanFileRow instanceof AddFile) {
+ Map<String, String> partitionValues =
+ InternalScanFileUtils.getPartitionValues(scanFileRow);
+ // List<Action> actionsForVersion =
+ // getChangesState().getActionsForVersion(versionNumber);
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ (AddFile) scanFileRow,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder().filesAdded(addedFiles.values()).build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ System.out.println("versionNumberAtLastSyncInstant: " + versionNumberAtLastSyncInstant);
+ // resetState(0, engine,table);
Review Comment:
This is the place which I have commented @vinishjail97 , call is happening inside this method
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2232854297
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Scan;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(Snapshot deltaSnapshot, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot, InternalSchema schema, boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+
+ Scan myScan = snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles = myScan.getScanFiles(engine);
Review Comment:
the above suggested changed worked for me, thanks @the-other-tim-brown
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Scan;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(Snapshot deltaSnapshot, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot, InternalSchema schema, boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+
+ Scan myScan = snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles = myScan.getScanFiles(engine);
+ this.dataFilesIterator =
+ Collections
+ .emptyIterator(); // Initialize the dataFilesIterator by iterating over the scan files
+ while (scanFiles.hasNext()) {
+ FilteredColumnarBatch scanFileColumnarBatch = scanFiles.next();
+ CloseableIterator<Row> scanFileRows = scanFileColumnarBatch.getRows();
+ while (scanFileRows.hasNext()) {
+ Row scanFileRow = scanFileRows.next();
Review Comment:
the above suggested changed worked for me, thanks @the-other-tim-brown
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2578056501
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ Map<String, String> values, List<InternalPartitionField> partitionFields) {
+ return partitionFields.stream()
+ .map(
+ partitionField -> {
+ PartitionTransformType partitionTransformType = partitionField.getTransformType();
+ String dateFormat =
+ partitionTransformType.isTimeBased()
+ ? getDateFormat(partitionTransformType)
+ : null;
+ String serializedValue = getSerializedPartitionValue(values, partitionField);
+ Object partitionValue =
+ convertFromDeltaPartitionValue(
+ serializedValue,
+ partitionField.getSourceField().getSchema().getDataType(),
+ partitionField.getTransformType(),
+ dateFormat);
+ return PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar(partitionValue))
+ .build();
+ })
+ .collect(toList(partitionFields.size()));
+ }
+
+ private String getSerializedPartitionValue(
+ Map<String, String> values, InternalPartitionField partitionField) {
+ if (partitionField.getPartitionFieldNames() == null
+ || partitionField.getPartitionFieldNames().isEmpty()) {
+ return values.getOrDefault(partitionField.getSourceField().getName(), null);
+ }
+ List<String> partitionFieldNames = partitionField.getPartitionFieldNames();
+ if (partitionFieldNames.size() == 1) {
+ return values.getOrDefault(partitionFieldNames.get(0), null);
+ }
+ return partitionFieldNames.stream()
+ .map(name -> values.get(name))
+ .collect(Collectors.joining("-"));
+ }
+
+ private String getGeneratedColumnName(InternalPartitionField internalPartitionField) {
+ return String.format(
+ DELTA_PARTITION_COL_NAME_FORMAT,
+ internalPartitionField.getTransformType().toString(),
+ internalPartitionField.getSourceField().getName());
+ }
+
+ private String getDateFormat(PartitionTransformType transformType) {
+ switch (transformType) {
+ case YEAR:
+ return DATE_FORMAT_FOR_YEAR;
+ case MONTH:
+ return DATE_FORMAT_FOR_MONTH;
+ case DAY:
+ return DATE_FORMAT_FOR_DAY;
+ case HOUR:
+ return DATE_FORMAT_FOR_HOUR;
+ default:
+ throw new PartitionSpecException("Invalid transform type");
+ }
+ }
+
+ private StructField getGeneratedField(InternalPartitionField internalPartitionField) {
+ String generatedExpression;
+ DataType dataType;
+ String currPartitionColumnName = getGeneratedColumnName(internalPartitionField);
+ switch (internalPartitionField.getTransformType()) {
+ case YEAR:
+ generatedExpression =
+ String.format(YEAR_FUNCTION, internalPartitionField.getSourceField().getPath());
+ dataType = IntegerType.INTEGER;
+ break;
+ case MONTH:
+ case HOUR:
+ generatedExpression =
+ String.format(
+ DATE_FORMAT_FUNCTION,
+ internalPartitionField.getSourceField().getPath(),
+ getDateFormat(internalPartitionField.getTransformType()));
+ dataType = IntegerType.INTEGER;
+ break;
+ case DAY:
+ generatedExpression =
+ String.format(CAST_FUNCTION, internalPartitionField.getSourceField().getPath());
+ dataType = DateType.DATE;
+ break;
+ case BUCKET:
+ generatedExpression =
+ String.format(
+ BUCKET_FUNCTION,
+ internalPartitionField.getSourceField().getPath(),
+ Integer.MAX_VALUE,
+ (int)
+ internalPartitionField
+ .getTransformOptions()
+ .get(InternalPartitionField.NUM_BUCKETS));
+ dataType = IntegerType.INTEGER;
+ break;
+ default:
+ throw new PartitionSpecException("Invalid transform type");
+ }
+ FieldMetadata partitionFieldMetadata =
+ FieldMetadata.builder().putString(DELTA_GENERATION_EXPRESSION, generatedExpression).build();
+ return new StructField(currPartitionColumnName, dataType, true, partitionFieldMetadata);
+ }
+
+ private void validate(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs,
+ Set<ParsedGeneratedExpr.GeneratedExprType> expectedTypesToBePresent) {
+ Set<String> sourceFields =
+ parsedGeneratedExprs.stream().map(expr -> expr.sourceColumn).collect(Collectors.toSet());
+ if (sourceFields.size() > 1) {
+ log.error(
+ String.format("Multiple source columns found for partition transform: %s", sourceFields));
+ throw new PartitionSpecException(
+ String.format("Multiple source columns found for partition transform: %s", sourceFields));
+ }
+ Set<ParsedGeneratedExpr.GeneratedExprType> actualTypesPresent =
+ parsedGeneratedExprs.stream()
+ .map(expr -> expr.generatedExprType)
+ .collect(Collectors.toSet());
+ if (!actualTypesPresent.equals(expectedTypesToBePresent)) {
+ log.error(
+ "Mismatched types present. Expected: "
+ + expectedTypesToBePresent
+ + ", Found: "
+ + actualTypesPresent);
+ throw new PartitionSpecException(
+ "Mismatched types present. Expected: "
+ + expectedTypesToBePresent
+ + ", Found: "
+ + actualTypesPresent);
+ }
+ }
+
+ private Map<String, String> convertScalaMapToJavaMap(
Review Comment:
This part was unused from anywhere in the code, so removed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2578058788
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(removeFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .build();
+ }
+
+ public FileFormat convertToFileFormat(String provider) {
+ if (provider.equals("parquet")) {
Review Comment:
this was coming as lowercase only. Anyhow I have corrected it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2538868077
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
it has been added back.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552732574
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.testutil.ColumnStatMapUtil.getColumnStats;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.*;
+
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.delta.kernel.expressions.Column;
+import io.delta.kernel.expressions.Literal;
+import io.delta.kernel.types.StringType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.testutil.ColumnStatMapUtil;
+
+public class TestDeltaKernelStatsExtractor {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ public void testDeltaStats() throws JsonProcessingException {
+ InternalSchema schema = ColumnStatMapUtil.getSchema();
+
+ List<ColumnStat> columnStats = getColumnStats();
+
+ String actualStats =
+ DeltaKernelStatsExtractor.getInstance().convertStatsToDeltaFormat(schema, 50L, columnStats);
+ Map<String, Object> actualStatsMap = MAPPER.readValue(actualStats, HashMap.class);
+ assertEquals(50, actualStatsMap.get("numRecords"));
+ Map<String, Object> minValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("minValues");
+ assertEquals(10, minValueStatsMap.get("long_field"));
+ assertEquals("a", minValueStatsMap.get("string_field"));
+ assertEquals(null, minValueStatsMap.get("null_string_field"));
+ assertEquals("2022-10-08 21:08:17", minValueStatsMap.get("timestamp_field"));
+ assertEquals("2022-10-08 21:08:17", minValueStatsMap.get("timestamp_micros_field"));
+ assertEquals(1.23, minValueStatsMap.get("float_field"));
+ assertEquals(1.23, minValueStatsMap.get("double_field"));
+ assertEquals(1.0, minValueStatsMap.get("decimal_field"));
+ // TOD0: Local timestamp depends on env where it is run, it is non determinstic and this has to
+ // be computed dynamically.
+ // assertEquals("2022-10-08 14:08:17", minValueStatsMap.get("local_timestamp_field"));
+ assertEquals("2019-10-12", minValueStatsMap.get("date_field"));
+ Map<String, Object> nestedMapInMinValueStatsMap =
+ (HashMap<String, Object>) minValueStatsMap.get("nested_struct_field");
+ assertEquals(500, nestedMapInMinValueStatsMap.get("nested_long_field"));
+
+ Map<String, Object> maxValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("maxValues");
+ assertEquals(20, maxValueStatsMap.get("long_field"));
+ assertEquals("c", maxValueStatsMap.get("string_field"));
+ assertEquals(null, maxValueStatsMap.get("null_string_field"));
+ assertEquals("2022-10-10 21:08:17", maxValueStatsMap.get("timestamp_field"));
+ assertEquals("2022-10-10 21:08:17", maxValueStatsMap.get("timestamp_micros_field"));
+ // TOD0: Local timestamp depends on env where it is run, it is non determinstic and this has to
+ // be computed dynamically.
+ // assertEquals("2022-10-10 14:08:17", maxValueStatsMap.get("local_timestamp_field"));
+ assertEquals("2020-10-12", maxValueStatsMap.get("date_field"));
+ assertEquals(6.54321, maxValueStatsMap.get("float_field"));
+ assertEquals(6.54321, maxValueStatsMap.get("double_field"));
+ assertEquals(2.0, maxValueStatsMap.get("decimal_field"));
+ Map<String, Object> nestedMapInMaxValueStatsMap =
+ (HashMap<String, Object>) maxValueStatsMap.get("nested_struct_field");
+ assertEquals(600, nestedMapInMaxValueStatsMap.get("nested_long_field"));
+
+ Map<String, Object> nullValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("nullCount");
+ assertEquals(4, nullValueStatsMap.get("long_field"));
+ assertEquals(1, nullValueStatsMap.get("string_field"));
+
+ assertEquals(3, nullValueStatsMap.get("null_string_field"));
+ assertEquals(105, nullValueStatsMap.get("timestamp_field"));
+ assertEquals(1, nullValueStatsMap.get("timestamp_micros_field"));
+ assertEquals(1, nullValueStatsMap.get("local_timestamp_field"));
+ assertEquals(250, nullValueStatsMap.get("date_field"));
+ assertEquals(2, nullValueStatsMap.get("float_field"));
+ assertEquals(3, nullValueStatsMap.get("double_field"));
+ assertEquals(1, nullValueStatsMap.get("decimal_field"));
+ Map<String, Object> nestedMapInNullCountMap =
+ (HashMap<String, Object>) nullValueStatsMap.get("nested_struct_field");
+ assertEquals(4, nestedMapInNullCountMap.get("nested_long_field"));
+ }
+
+ private Map<Column, Literal> parseValues(JsonNode valuesNode) {
+ Map<Column, Literal> values = new HashMap<>();
+ if (valuesNode == null || valuesNode.isNull()) {
+ return values;
+ }
+
+ Iterator<Map.Entry<String, JsonNode>> fields = valuesNode.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> entry = fields.next();
+ String columnName = entry.getKey();
+ JsonNode valueNode = entry.getValue();
+ values.put(new Column(columnName), convertToLiteral(valueNode));
+ }
+ return values;
+ }
+
+ private Literal convertToLiteral(JsonNode valueNode) {
+ System.out.println("ValueNode: " + valueNode);
Review Comment:
fixed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2577411376
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(removeFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .build();
+ }
+
+ public FileFormat convertToFileFormat(String provider) {
+ if (provider.equals("parquet")) {
Review Comment:
is provider always lowercase or uppercase? If not I think we should do `toLowerCase` on it when doing these string comparisons
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2426985706
##########
xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+import org.apache.xtable.kernel.DeltaKernelConversionSourceProvider;
+import org.apache.xtable.kernel.DeltaKernelPartitionExtractor;
+import org.apache.xtable.kernel.DeltaKernelSchemaExtractor;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.upsertRows(rows.subList(0, 20));
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(200L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testsShowingVacuumHasNoEffectOnIncrementalSync() {
+ boolean isPartitioned = true;
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // Insert 50 rows to 2018 partition.
+ List<Row> commit1Rows = testSparkDeltaTable.insertRowsForPartition(50, 2018);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ InternalSnapshot snapshotAfterCommit1 = conversionSource.getCurrentSnapshot();
+ List<String> allActivePaths = ValidationTestHelper.getAllFilePaths(snapshotAfterCommit1);
+ assertEquals(1, allActivePaths.size());
+ String activePathAfterCommit1 = allActivePaths.get(0);
+
+ // Upsert all rows inserted before, so all files are replaced.
+ testSparkDeltaTable.upsertRows(commit1Rows.subList(0, 50));
+
+ // Insert 50 rows to different (2020) partition.
+ testSparkDeltaTable.insertRowsForPartition(50, 2020);
+
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ conversionSource = conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ CommitsBacklog<Long> instantCurrentCommitState =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(timestamp1)));
+ // // Table doesn't have instant of this older commit, hence it is not safe.
+ Instant instantAsOfHourAgo = Instant.now().minus(1, ChronoUnit.HOURS);
+ assertFalse(conversionSource.isIncrementalSyncSafeFrom(instantAsOfHourAgo));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testAddColumns(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, true);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(150L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testDropPartition() {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(tableName, tempDir, sparkSession, "yearOfBirth", false);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> rows1 = testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> allRows = new ArrayList<>();
+ allRows.addAll(rows);
+ allRows.addAll(rows1);
+
+ Map<Integer, List<Row>> rowsByPartition = testSparkDeltaTable.getRowsByPartition(allRows);
+ Integer partitionValueToDelete = rowsByPartition.keySet().stream().findFirst().get();
+ testSparkDeltaTable.deletePartition(partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ // Insert few records for deleted partition again to make it interesting.
+ testSparkDeltaTable.insertRowsForPartition(20, partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(
+ 120 - rowsByPartition.get(partitionValueToDelete).size(), testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ validateDeltaPartitioning(internalSnapshot);
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ void testConvertFromDeltaPartitionSinglePartition() {
Review Comment:
@the-other-tim-brown I have added this unit test, Please confirm if this looks fine I will add the others too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2410774652
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.*;
+
+import io.delta.kernel.types.*;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaPartitionExtractor;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
Review Comment:
I'm not asking for a functional test, I am asking for unit testing. We want high coverage with the testing of these key components.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2483786141
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java:
##########
@@ -41,22 +37,10 @@
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.schema.SchemaUtils;
-/**
- * Converts between Delta and InternalTable schemas. Some items to be aware of:
- *
- * <ul>
- * <li>Delta schemas are represented as Spark StructTypes which do not have enums so the enum
Review Comment:
@vaibhavk1992 please follow up on this
##########
xtable-core/src/test/java/org/apache/xtable/kernel/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+import org.apache.xtable.kernel.DeltaKernelConversionSourceProvider;
+import org.apache.xtable.kernel.DeltaKernelPartitionExtractor;
+import org.apache.xtable.kernel.DeltaKernelSchemaExtractor;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.upsertRows(rows.subList(0, 20));
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(200L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testsShowingVacuumHasNoEffectOnIncrementalSync() {
+ boolean isPartitioned = true;
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // Insert 50 rows to 2018 partition.
+ List<Row> commit1Rows = testSparkDeltaTable.insertRowsForPartition(50, 2018);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ InternalSnapshot snapshotAfterCommit1 = conversionSource.getCurrentSnapshot();
+ List<String> allActivePaths = ValidationTestHelper.getAllFilePaths(snapshotAfterCommit1);
+ assertEquals(1, allActivePaths.size());
+ String activePathAfterCommit1 = allActivePaths.get(0);
+
+ // Upsert all rows inserted before, so all files are replaced.
+ testSparkDeltaTable.upsertRows(commit1Rows.subList(0, 50));
+
+ // Insert 50 rows to different (2020) partition.
+ testSparkDeltaTable.insertRowsForPartition(50, 2020);
+
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ conversionSource = conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ CommitsBacklog<Long> instantCurrentCommitState =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(timestamp1)));
+ // // Table doesn't have instant of this older commit, hence it is not safe.
+ Instant instantAsOfHourAgo = Instant.now().minus(1, ChronoUnit.HOURS);
+ assertFalse(conversionSource.isIncrementalSyncSafeFrom(instantAsOfHourAgo));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testAddColumns(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, true);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(150L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testDropPartition() {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(tableName, tempDir, sparkSession, "yearOfBirth", false);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> rows1 = testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> allRows = new ArrayList<>();
+ allRows.addAll(rows);
+ allRows.addAll(rows1);
+
+ Map<Integer, List<Row>> rowsByPartition = testSparkDeltaTable.getRowsByPartition(allRows);
+ Integer partitionValueToDelete = rowsByPartition.keySet().stream().findFirst().get();
+ testSparkDeltaTable.deletePartition(partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ // Insert few records for deleted partition again to make it interesting.
+ testSparkDeltaTable.insertRowsForPartition(20, partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(
+ 120 - rowsByPartition.get(partitionValueToDelete).size(), testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ validateDeltaPartitioning(internalSnapshot);
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ void testConvertFromDeltaPartitionSinglePartition() {
Review Comment:
Looks good!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3369251956
Below is the summary of the difference between two schemes (Delta vs Kernel) also added the what remains the difference between two.
### Comparison of Schema Responses: Delta Kernel vs Delta Log
This document outlines the differences in schema responses when using **Delta Kernel** and **Delta Log** APIs to retrieve changes in a Delta table. The comparison highlights the structure and format of the responses, providing insights into how the two approaches differ.
#### **Delta Kernel Schema Response**
When using the `DeltaKernelIncrementalChangesState` class to retrieve changes, the response is in the form of a **row of columnar batch type**. Each row is represented as an object of the `io.delta.kernel.data.Row` interface, which provides methods to access individual fields. The response is minimalistic and focuses on the raw data representation.
##### **Sample Output**
1 row is an object ==> io.delta.kernel.internal.data.ColumnarBatchRow@20c03e47
##### **Key Characteristics**
1. **Row Representation**: Each row is an instance of `ColumnarBatchRow`, which provides methods to access fields like `getLong`, `getString`, [etc](https://docs.delta.io/api/latest/java/kernel/io/delta/kernel/data/row).
2. **Minimal Metadata**: The response contains only the essential fields (e.g., `version`, `timestamp`, `commitInfo`).
3. **Raw Data**: The schema is not enriched with additional metadata or actions; it is a direct representation of the data in the columnar batch.
##### **Use Case**
This format is suitable for low-level data processing where the focus is on performance and accessing raw data.
#### **Delta Log Schema Response**
When using the `DeltaLog.getChanges` method, the response is a **tuple** containing the version number and a list of actions. The actions include detailed metadata about the changes, such as `CommitInfo` and `AddFile`.
Sample Output from delta table changes
(2,
Vector(
CommitInfo(None, 2025-08-15 15:00:46.05, None, None, WRITE, Map(mode -> Append, partitionBy -> []), None, None, None, Some(1), Some(Serializable), Some(true), Some(Map(numFiles -> 1, numOutputRows -> 50, numOutputBytes -> 10226)), None, None, Some(Apache-Spark/3.4.2 Delta-Lake/2.4.0), Some(cf7b1472-4c68-4f89-aa97-c8f16512ecfc)),
AddFile(part-00000-e8eeadc8-4e26-46a7-8c61-0bf60e5e7ada-c000.snappy.parquet, Map(), 10226, 1755250246045, true, {"numRecords":50,"minValues":{"id":51,"firstName":"0WI98","lastName":"08VkW","gender":"Female","birthDate":"2013-02-16T21:18:43.000+05:30","level":"ERROR","date_field":"2025-08-15","timestamp_field":"2025-08-15T15:00:45.884+05:30","double_field":0.018425752795049544,"float_field":0.109567106,"long_field":-8844008067348082419,"record_field":{"nested_int":-2060061976}},"maxValues":{"id":100,"firstName":"xZnER","lastName":"ymLQw","gender":"Male","birthDate":"2023-08-07T15:06:55.000+05:30","level":"WARN","date_field":"2025-08-15","timestamp_field":"2025-08-15T15:00:45.885+05:30","double_field":0.9914942463945434,"float_field":0.9841615,"long_field":8775924211265194460,"record_field":{"nested_int":1923869027}},"nullCount":{"id":0,"firstName":25,"lastName":23,"gender":0,"birthDate":0,"level":0,"boolean_field":28,"date_field":24,"timestamp_field":26,"double_field":25,"float_f
ield":28,"long_field":25,"binary_field":32,"primitive_map":22,"record_map":25,"primitive_list":28,"record_list":29,"record_field":{"nested_int":28}}}, null, null)
))
Key Characteristics
1. **Rich Metadata**: The response includes detailed metadata such as `CommitInfo` (e.g., operation type, timestamp, and user metadata) and `AddFile` (e.g., file path, size, and statistics).
2. **Structured Actions**: Each action is represented as a specific object (e.g., `CommitInfo`, `AddFile`), making it easier to interpret the changes.
3. **Verbose Output**: The response is more verbose, providing a comprehensive view of the changes.
This issue is currently in **blocked** state. I raised it with delta team quite a few time but no response over it.
https://delta-users.slack.com/archives/C04TRPG3LHZ/p1758730559515289
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559723658
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
I have used SchemaExtractorException exception present in xtable repo.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559723658
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
I have used SchemaExtractorException exception present in xtable repo. Let me know if it works.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ java.util.Map<String, String> scalaMap = partitionValues;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559841407
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552731666
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+ // The immutable map from Java to Scala is not working, need to
Review Comment:
fixed it.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ System.out.println("if coming");
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ System.out.println("else coming");
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ scala.collection.Map<String, String> values, List<InternalPartitionField> partitionFields) {
Review Comment:
fixed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2534715436
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.util.*;
+
+import io.delta.kernel.types.DataType;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
+
+ private static final String DELTA_COLUMN_MAPPING_ID = "delta.columnMapping.id";
+ private static final DeltaKernelSchemaExtractor INSTANCE = new DeltaKernelSchemaExtractor();
+ private static final Map<InternalSchema.MetadataKey, Object>
+ DEFAULT_TIMESTAMP_PRECISION_METADATA =
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
+
+ public static DeltaKernelSchemaExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalSchema toInternalSchema_v2(StructType structType) {
Review Comment:
This has been addressed in the most recent code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2535047609
##########
xtable-core/src/test/resources/junit-platform.properties:
##########
@@ -14,6 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-junit.jupiter.execution.parallel.enabled=true
+junit.jupiter.execution.parallel.enabled=false
Review Comment:
reverted it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vinishjail97 (via GitHub)" <gi...@apache.org>.
vinishjail97 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2236824550
##########
xtable-utilities/src/test/resources/my_config.yaml:
##########
@@ -19,6 +19,6 @@ targetFormats:
- DELTA
datasets:
-
- tableBasePath: /Desktop/opensource/iceberg/warehouse/demo/nyc/taxis
- tableDataPath: /Desktop/opensource/iceberg/warehouse/demo/nyc/taxis/data
+ tableBasePath: /Users/vaibhakumar/Desktop/opensource/iceberg/warehouse/demo/nyc/taxis
Review Comment:
This one can be reverted too?
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
Review Comment:
Let's move new DeltaKernel* classes to org.apache.xtable.kernel?
Any reason for having kernel in a new package outside delta? We might rename to delta-kernel or something if we keep it outside.
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
@vaibhavk1992 I think if you rebase with latest main branch you shouldn't see those failures.
##########
pom.xml:
##########
@@ -713,7 +713,7 @@
</executions>
<configuration>
<skip>${skipUTs}</skip>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
Review Comment:
Why revert this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559422880
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,20 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-defaults</artifactId>
+ <version>${delta.kernel.version}</version>
Review Comment:
The dependency issue was resolved too, the issue was happening because kernel also comes up with it's hadoop-common, so just had to change the scope of kernel one to provided
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2582267216
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final CloseableIterator<FilteredColumnarBatch> scanFiles;
+ private final FileFormat fileFormat;
+ private final Table table;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private final boolean includeColumnStats;
+
+ private CloseableIterator<Row> currentFileRows;
+ private InternalDataFile nextFile;
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ this.includeColumnStats = includeColumnStats;
+ this.table = table;
+ this.fields = schema.getFields();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ this.scanFiles = myScan.getScanFiles(engine, includeColumnStats);
+
+ // Initialize first element
+ this.nextFile = computeNext();
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ if (currentFileRows != null) {
+ currentFileRows.close();
+ }
+ } finally {
+ scanFiles.close();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return nextFile != null;
+ }
+
+ @Override
+ public InternalDataFile next() {
+ InternalDataFile current = nextFile;
+ nextFile = computeNext();
+ return current;
+ }
+
+ private InternalDataFile computeNext() {
+ try {
+ while (true) {
Review Comment:
fixed the code and removed the while true condition.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown merged PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2535070511
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,19 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>4.0.0</version>
Review Comment:
changes have been added to the root pom.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelConversionSourceProvider.java:
##########
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.conversion.ConversionSourceProvider;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+
+public class DeltaKernelConversionSourceProvider extends ConversionSourceProvider<Long> {
+ @Override
+ public DeltaKernelConversionSource getConversionSourceInstance(SourceTable sourceTable) {
+ Configuration hadoopConf = new Configuration();
Review Comment:
I have removed the redundant config.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java:
##########
@@ -41,22 +37,10 @@
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.schema.SchemaUtils;
-/**
- * Converts between Delta and InternalTable schemas. Some items to be aware of:
- *
- * <ul>
- * <li>Delta schemas are represented as Spark StructTypes which do not have enums so the enum
Review Comment:
changes are done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2557245771
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,20 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-defaults</artifactId>
+ <version>${delta.kernel.version}</version>
Review Comment:
You should add the dependencies under the `dependencyManagement` section. This enforces consistent versions throughout the project instead of letting each module declare its own version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2557196268
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,20 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-defaults</artifactId>
+ <version>${delta.kernel.version}</version>
Review Comment:
Declare these dependency versions in the parent pom instead
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2566347453
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
You can update the `delta-kernel-defaults` to exclude the `hadoop-client-runtime`
```
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client-runtime</artifactId>
</exclusion>
</exclusions>```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2566548729
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
@vaibhavk1992 @the-other-tim-brown When i tried reproducing by setting this to `compile` I hit the following errors, in xtable service. I am not sure why this breaks from having delta kernel as compile scope.
```
[ERROR] COMPILATION ERROR :
[INFO] -------------------------------------------------------------
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[35,30] package org.apache.hadoop.conf does not exist
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[75,17] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[105,11] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[140,72] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[175,7] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
```
This `Configuration` class seems to come from `hadoop-common`
<img width="313" height="233" alt="Screenshot 2025-11-26 at 1 35 32 PM" src="https://github.com/user-attachments/assets/fb780784-cf28-42b9-b924-10aa87d63469" />
In order to get past the build exception I included in the `xtable-service` pom and was able to get build to pass.
```
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
```
I am ok with the exclusion idea that @the-other-tim-brown mentioned if it allows the build to pass. but wanted to know if this was more ideal instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565868586
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
Which command can I run to reproduce this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2535038561
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ this.dataFilesIterator =
+ Collections
+ .emptyIterator(); // Initialize the dataFilesIterator by iterating over the scan files
Review Comment:
I have resolved it as per new changes.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ System.err.println(
Review Comment:
I have used logger now as part of latest changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552734074
##########
xtable-core/src/test/java/org/apache/xtable/kernel/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,734 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
Review Comment:
fixed it.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ System.out.println("if coming");
Review Comment:
fixed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552733693
##########
xtable-core/src/test/java/org/apache/xtable/DeltaTableKernel.java:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable;
+
+// import org.junit.jupiter.api.Test;
+//
+import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.delta.kernel.*;
+import io.delta.kernel.data.ColumnVector;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.*;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.data.ScanStateRow;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+public class DeltaTableKernel {
+ private static final Logger logger = LoggerFactory.getLogger(DeltaTableKernel.class);
+
+ @Test
+ public void readDeltaKernel() throws IOException {
+ String myTablePath =
+ "/Users/vaibhakumar/Desktop/opensource/iceberg/warehouse/demo/nyc/taxis"; // fully qualified
Review Comment:
Removed it, wasn't needed.
##########
pom.xml:
##########
@@ -724,7 +725,7 @@
</executions>
<configuration>
<skip>${skipUTs}</skip>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <redirectTestOutputToFile>false</redirectTestOutputToFile>
Review Comment:
fixed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552744343
##########
xtable-service/pom.xml:
##########
@@ -60,7 +60,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
-
+ <dependency>
Review Comment:
Somehow hadoop common library import is not working when present in parent POM, so after checking with Rahil he suggested that we can have it in the x-table service pom. Otherwise xtable-service module doesn't build, it gave hadoop-common errors.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559445340
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Slf4j
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ log.error(
+ "Error checking if incremental sync is safe from " + instant + ": " + e.getMessage());
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559681587
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ java.util.Map<String, String> values, List<InternalPartitionField> partitionFields) {
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565841550
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
@the-other-tim-brown Regarding the kernel scope in POM, if I keep it compile it gives me those same errors package doesn't exist. Maven resolves hadoop-common with compile scope (from Delta Kernel), overriding existing provided declaration so I kept it provided.
Let me know if there is any other solution that may work here or if we can keep the scope as it is for this PR.
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Slf4j
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ log.error("Error checking if incremental sync is safe from " + instant + ": " + e);
Review Comment:
Done
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.exception.SchemaExtractorException;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+ List<StructField> partitionFieldSchemas =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFieldSchemas);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
Done, removed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559418156
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,20 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-defaults</artifactId>
+ <version>${delta.kernel.version}</version>
Review Comment:
Moved to parent pom under dependency management.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2577574802
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ Map<String, String> values, List<InternalPartitionField> partitionFields) {
+ return partitionFields.stream()
+ .map(
+ partitionField -> {
+ PartitionTransformType partitionTransformType = partitionField.getTransformType();
+ String dateFormat =
+ partitionTransformType.isTimeBased()
+ ? getDateFormat(partitionTransformType)
+ : null;
+ String serializedValue = getSerializedPartitionValue(values, partitionField);
+ Object partitionValue =
+ convertFromDeltaPartitionValue(
+ serializedValue,
+ partitionField.getSourceField().getSchema().getDataType(),
+ partitionField.getTransformType(),
+ dateFormat);
+ return PartitionValue.builder()
+ .partitionField(partitionField)
+ .range(Range.scalar(partitionValue))
+ .build();
+ })
+ .collect(toList(partitionFields.size()));
+ }
+
+ private String getSerializedPartitionValue(
+ Map<String, String> values, InternalPartitionField partitionField) {
+ if (partitionField.getPartitionFieldNames() == null
+ || partitionField.getPartitionFieldNames().isEmpty()) {
+ return values.getOrDefault(partitionField.getSourceField().getName(), null);
+ }
+ List<String> partitionFieldNames = partitionField.getPartitionFieldNames();
+ if (partitionFieldNames.size() == 1) {
+ return values.getOrDefault(partitionFieldNames.get(0), null);
+ }
+ return partitionFieldNames.stream()
+ .map(name -> values.get(name))
+ .collect(Collectors.joining("-"));
+ }
+
+ private String getGeneratedColumnName(InternalPartitionField internalPartitionField) {
+ return String.format(
+ DELTA_PARTITION_COL_NAME_FORMAT,
+ internalPartitionField.getTransformType().toString(),
+ internalPartitionField.getSourceField().getName());
+ }
+
+ private String getDateFormat(PartitionTransformType transformType) {
+ switch (transformType) {
+ case YEAR:
+ return DATE_FORMAT_FOR_YEAR;
+ case MONTH:
+ return DATE_FORMAT_FOR_MONTH;
+ case DAY:
+ return DATE_FORMAT_FOR_DAY;
+ case HOUR:
+ return DATE_FORMAT_FOR_HOUR;
+ default:
+ throw new PartitionSpecException("Invalid transform type");
+ }
+ }
+
+ private StructField getGeneratedField(InternalPartitionField internalPartitionField) {
+ String generatedExpression;
+ DataType dataType;
+ String currPartitionColumnName = getGeneratedColumnName(internalPartitionField);
+ switch (internalPartitionField.getTransformType()) {
+ case YEAR:
+ generatedExpression =
+ String.format(YEAR_FUNCTION, internalPartitionField.getSourceField().getPath());
+ dataType = IntegerType.INTEGER;
+ break;
+ case MONTH:
+ case HOUR:
+ generatedExpression =
+ String.format(
+ DATE_FORMAT_FUNCTION,
+ internalPartitionField.getSourceField().getPath(),
+ getDateFormat(internalPartitionField.getTransformType()));
+ dataType = IntegerType.INTEGER;
+ break;
+ case DAY:
+ generatedExpression =
+ String.format(CAST_FUNCTION, internalPartitionField.getSourceField().getPath());
+ dataType = DateType.DATE;
+ break;
+ case BUCKET:
+ generatedExpression =
+ String.format(
+ BUCKET_FUNCTION,
+ internalPartitionField.getSourceField().getPath(),
+ Integer.MAX_VALUE,
+ (int)
+ internalPartitionField
+ .getTransformOptions()
+ .get(InternalPartitionField.NUM_BUCKETS));
+ dataType = IntegerType.INTEGER;
+ break;
+ default:
+ throw new PartitionSpecException("Invalid transform type");
+ }
+ FieldMetadata partitionFieldMetadata =
+ FieldMetadata.builder().putString(DELTA_GENERATION_EXPRESSION, generatedExpression).build();
+ return new StructField(currPartitionColumnName, dataType, true, partitionFieldMetadata);
+ }
+
+ private void validate(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs,
+ Set<ParsedGeneratedExpr.GeneratedExprType> expectedTypesToBePresent) {
+ Set<String> sourceFields =
+ parsedGeneratedExprs.stream().map(expr -> expr.sourceColumn).collect(Collectors.toSet());
+ if (sourceFields.size() > 1) {
+ log.error(
+ String.format("Multiple source columns found for partition transform: %s", sourceFields));
+ throw new PartitionSpecException(
+ String.format("Multiple source columns found for partition transform: %s", sourceFields));
+ }
+ Set<ParsedGeneratedExpr.GeneratedExprType> actualTypesPresent =
+ parsedGeneratedExprs.stream()
+ .map(expr -> expr.generatedExprType)
+ .collect(Collectors.toSet());
+ if (!actualTypesPresent.equals(expectedTypesToBePresent)) {
+ log.error(
+ "Mismatched types present. Expected: "
+ + expectedTypesToBePresent
+ + ", Found: "
+ + actualTypesPresent);
+ throw new PartitionSpecException(
+ "Mismatched types present. Expected: "
+ + expectedTypesToBePresent
+ + ", Found: "
+ + actualTypesPresent);
+ }
+ }
+
+ private Map<String, String> convertScalaMapToJavaMap(
Review Comment:
nit: Do we really need this method, can we just call directly the `JavaConverters.mapAsJavaMapConverter`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3598009826
@rahil-c As discussed `ITConversionController` would be part of the follow PR. I have resolved the 2 comments from you as well. If things are looking good can you and @the-other-tim-brown please approve the changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3597316671
@vaibhavk1992 Thanks for pushing on this, I took a brief scan and the PR looks close to landing. My only main question is around testing, did you happen to test conversion end to end using your delta kernel source to hudi or to iceberg?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176555979
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelConversionSourceProvider.java:
##########
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.conversion.ConversionSourceProvider;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+
+public class DeltaKernelConversionSourceProvider extends ConversionSourceProvider<Long> {
+ @Override
+ public DeltaKernelConversionSource getConversionSourceInstance(SourceTable sourceTable) {
+ Configuration hadoopConf = new Configuration();
Review Comment:
Is there any reason why you are creating a new hadoopConf, can you instead use the hadoopConf from the parent class similar to what `DeltaConversionSourceProvider` does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176574684
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java:
##########
@@ -41,22 +37,10 @@
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.schema.SchemaUtils;
-/**
- * Converts between Delta and InternalTable schemas. Some items to be aware of:
- *
- * <ul>
- * <li>Delta schemas are represented as Spark StructTypes which do not have enums so the enum
Review Comment:
I think we can live this file as is right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552734996
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.kernel.DeltaKernelPartitionExtractor.DELTA_GENERATION_EXPRESSION;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+
+public class TestDeltaKernelPartitionExtractor {
+ private static final Map<String, StructField> STRUCT_FIELD_MAP =
+ new HashMap<String, StructField>() {
+ {
+ put("id", new StructField("id", IntegerType.INTEGER, false));
+ put("firstName", new StructField("firstName", StringType.STRING, false));
+ put("gender", new StructField("gender", StringType.STRING, false));
+ put("birthDate", new StructField("birthDate", TimestampType.TIMESTAMP, false));
+ put(
+ "dateOfBirth",
+ new StructField(
+ "dateOfBirth",
+ DateType.DATE,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "CAST(birthDate AS DATE)")
+ .build()));
+
+ put(
+ "dateFmt",
+ new StructField(
+ "dateFmt",
+ StringType.STRING,
+ false,
+ FieldMetadata.builder()
+ .putString(
+ "delta.generationExpression", "DATE_FORMAT(birthDate, 'yyyy-MM-dd-HH')")
+ .build()));
+
+ put(
+ "yearOfBirth",
+ new StructField(
+ "yearOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "YEAR(birthDate)")
+ .build()));
+ put(
+ "monthOfBirth",
+ new StructField(
+ "monthOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "MONTH(birthDate)")
+ .build()));
+
+ put(
+ "dayOfBirth",
+ new StructField(
+ "dayOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "DAY(birthDate)")
+ .build()));
+
+ put(
+ "hourOfBirth",
+ new StructField(
+ "hourOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "HOUR(birthDate)")
+ .build()));
+ }
+ };
+ private static final InternalSchema TIMESTAMP_SCHEMA =
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .metadata(
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
+ .build();
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+ private final DeltaKernelSchemaExtractor deltaKernelSchemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ @Test
+ public void testUnpartitionedTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, new StructType());
+ assertTrue(internalPartitionFields.isEmpty());
+ }
+
+ @Test
+ public void testSimplePartitionedTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("gender"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("gender")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDatePartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateOfBirth"));
+
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.DAY)
+ .partitionFieldNames(Collections.singletonList("dateOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDateFormatPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateFmt"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.HOUR)
+ .partitionFieldNames(Collections.singletonList("dateFmt"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.YEAR)
+ .partitionFieldNames(Collections.singletonList("yearOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearAndSimpleCombinedPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth", "id"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.YEAR)
+ .partitionFieldNames(Collections.singletonList("yearOfBirth"))
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearMonthDayHourPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(
+ Arrays.asList(
+ "id",
+ "firstName",
+ "gender",
+ "birthDate",
+ "yearOfBirth",
+ "monthOfBirth",
+ "dayOfBirth",
+ "hourOfBirth"));
+ StructType partitionSchema =
+ getSchemaWithFields(
+ Arrays.asList("yearOfBirth", "monthOfBirth", "dayOfBirth", "hourOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .partitionFieldNames(
+ Arrays.asList("yearOfBirth", "monthOfBirth", "dayOfBirth", "hourOfBirth"))
+ .transformType(PartitionTransformType.HOUR)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ // Test for preserving order of partition columns.
+ @Test
+ public void testCombinationOfPlainAndGeneratedColumns() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
+ StructType partitionSchema =
+ getSchemaWithFields(Arrays.asList("id", "dateFmt", "gender", "dateOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.HOUR)
+ .partitionFieldNames(Collections.singletonList("dateFmt"))
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("gender")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.DAY)
+ .partitionFieldNames(Collections.singletonList("dateOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDateFormatGeneratedPartitionValueExtraction() {
+ // date_partition_column is generated in the table as DATE_FORMAT(some_date_column,
+ // 'yyyy-MM-dd-HH')
+ // where some_date_column is of timestamp type.
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("date_partition_column", "2013-08-20-10");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(Collections.singletonList("date_partition_column"))
+ .transformType(PartitionTransformType.HOUR)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar(1376992800000L);
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ public void testSimplePartitionValueExtraction() {
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("partition_column2", "partition_value2");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column2")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar("partition_value2");
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ public void testYearMonthDayHourGeneratedPartitionValueExtraction() {
+ // year, month and day are generated in the table as based on some_date_column which is of
+ // timestamp type.
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("year_partition_column", "2013");
+ put("month_partition_column", "8");
+ put("day_partition_column", "20");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(
+ Arrays.asList(
+ "year_partition_column", "month_partition_column", "day_partition_column"))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar(1376956800000L);
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ void convertBucketPartition() {
+ InternalPartitionField internalPartitionField =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.BUCKET)
+ .transformOptions(Collections.singletonMap(InternalPartitionField.NUM_BUCKETS, 5))
+ .build();
+ System.out.println("internalPartitionField" + internalPartitionField);
Review Comment:
fixed it.
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,864 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+
+public class TestDeltaKernelSchemaExtractor {
+ @Test
+ public void testPrimitiveTypes() {
+ Map<InternalSchema.MetadataKey, Object> decimalMetadata = new HashMap<>();
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 10);
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 2);
+
+ InternalSchema internalSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .isNullable(false)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("requiredBoolean")
+ .schema(
+ InternalSchema.builder()
+ .name("boolean")
+ .dataType(InternalType.BOOLEAN)
+ .isNullable(false)
+ .comment("requiredBooleanComment")
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalBoolean")
+ .schema(
+ InternalSchema.builder()
+ .name("boolean")
+ .dataType(InternalType.BOOLEAN)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredInt")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalInt")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredLong")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalLong")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDouble")
+ .schema(
+ InternalSchema.builder()
+ .name("double")
+ .dataType(InternalType.DOUBLE)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDouble")
+ .schema(
+ InternalSchema.builder()
+ .name("double")
+ .dataType(InternalType.DOUBLE)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredFloat")
+ .schema(
+ InternalSchema.builder()
+ .name("float")
+ .dataType(InternalType.FLOAT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalFloat")
+ .schema(
+ InternalSchema.builder()
+ .name("float")
+ .dataType(InternalType.FLOAT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredString")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalString")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredBytes")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+ .dataType(InternalType.BYTES)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalBytes")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+ .dataType(InternalType.BYTES)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDate")
+ .schema(
+ InternalSchema.builder()
+ .name("date")
+ .dataType(InternalType.DATE)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDate")
+ .schema(
+ InternalSchema.builder()
+ .name("date")
+ .dataType(InternalType.DATE)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDecimal")
+ .schema(
+ InternalSchema.builder()
+ .name("decimal")
+ .dataType(InternalType.DECIMAL)
+ .isNullable(false)
+ .metadata(decimalMetadata)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDecimal")
+ .schema(
+ InternalSchema.builder()
+ .name("decimal")
+ .dataType(InternalType.DECIMAL)
+ .isNullable(true)
+ .metadata(decimalMetadata)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .build();
+ io.delta.kernel.types.StructType structRepresentation =
Review Comment:
fixed it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2552744343
##########
xtable-service/pom.xml:
##########
@@ -60,7 +60,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
-
+ <dependency>
Review Comment:
hadoop common library import is not working when present in parent POM, so after checking with Rahil he suggested that we can have it in the x-table service pom. Otherwise xtable-service module doesn't build, it gave hadoop-common errors.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559415076
##########
xtable-service/pom.xml:
##########
@@ -60,7 +60,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
-
+ <dependency>
Review Comment:
this has been removed and issue was resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2557194456
##########
xtable-service/pom.xml:
##########
@@ -60,7 +60,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
-
+ <dependency>
Review Comment:
This is unrelated to this change though so we should pull it in another PR where the problem is clear. I am able to build locally without this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3583024549
@vaibhavk1992 make sure to fill out the PR template
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559441607
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.util.*;
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2577763275
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(removeFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .build();
+ }
+
+ public FileFormat convertToFileFormat(String provider) {
+ if (provider.equals("parquet")) {
Review Comment:
or use `equalsIgnoreCase`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2534717125
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+
+ System.out.println("Kernelschema: " + schema);
Review Comment:
This has been addressed in the most recent code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2535042502
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
Review Comment:
Resolved it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2538870455
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2557213250
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,20 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-defaults</artifactId>
+ <version>${delta.kernel.version}</version>
Review Comment:
Yes, It is present in parent POM already
<img width="506" height="64" alt="image" src="https://github.com/user-attachments/assets/60081c6d-1fd8-4621-b8c7-10c81a2f8d44" />
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559430913
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
Review Comment:
I had changed it but dont't know why some of the changes I made are not showing up correctly. Will change them again.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2577411376
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(removeFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .build();
+ }
+
+ public FileFormat convertToFileFormat(String provider) {
+ if (provider.equals("parquet")) {
Review Comment:
is provider always lowercase or uppercase? If not I think we should do `toLowerCase` on it when doing these string comparisons. This comment applies for in general in the pr
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559844758
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ java.util.Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ java.util.Map<String, String> scalaMap = partitionValues;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2457037175
##########
xtable-core/src/test/java/org/apache/xtable/kernel/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+import org.apache.xtable.kernel.DeltaKernelConversionSourceProvider;
+import org.apache.xtable.kernel.DeltaKernelPartitionExtractor;
+import org.apache.xtable.kernel.DeltaKernelSchemaExtractor;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.upsertRows(rows.subList(0, 20));
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(200L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testsShowingVacuumHasNoEffectOnIncrementalSync() {
+ boolean isPartitioned = true;
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // Insert 50 rows to 2018 partition.
+ List<Row> commit1Rows = testSparkDeltaTable.insertRowsForPartition(50, 2018);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ InternalSnapshot snapshotAfterCommit1 = conversionSource.getCurrentSnapshot();
+ List<String> allActivePaths = ValidationTestHelper.getAllFilePaths(snapshotAfterCommit1);
+ assertEquals(1, allActivePaths.size());
+ String activePathAfterCommit1 = allActivePaths.get(0);
+
+ // Upsert all rows inserted before, so all files are replaced.
+ testSparkDeltaTable.upsertRows(commit1Rows.subList(0, 50));
+
+ // Insert 50 rows to different (2020) partition.
+ testSparkDeltaTable.insertRowsForPartition(50, 2020);
+
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ conversionSource = conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ CommitsBacklog<Long> instantCurrentCommitState =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(timestamp1)));
+ // // Table doesn't have instant of this older commit, hence it is not safe.
+ Instant instantAsOfHourAgo = Instant.now().minus(1, ChronoUnit.HOURS);
+ assertFalse(conversionSource.isIncrementalSyncSafeFrom(instantAsOfHourAgo));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testAddColumns(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, true);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(150L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testDropPartition() {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(tableName, tempDir, sparkSession, "yearOfBirth", false);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> rows1 = testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> allRows = new ArrayList<>();
+ allRows.addAll(rows);
+ allRows.addAll(rows1);
+
+ Map<Integer, List<Row>> rowsByPartition = testSparkDeltaTable.getRowsByPartition(allRows);
+ Integer partitionValueToDelete = rowsByPartition.keySet().stream().findFirst().get();
+ testSparkDeltaTable.deletePartition(partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ // Insert few records for deleted partition again to make it interesting.
+ testSparkDeltaTable.insertRowsForPartition(20, partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(
+ 120 - rowsByPartition.get(partitionValueToDelete).size(), testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ validateDeltaPartitioning(internalSnapshot);
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ void testConvertFromDeltaPartitionSinglePartition() {
Review Comment:
@the-other-tim-brown as you suggested referring TestDeltaPartitionExtractor I have created the [TestDeltaKernelPartitionExtractor](https://github.com/apache/incubator-xtable/pull/729/files#diff-6e55e46c02e67c94611325b57c386659ec14a42b344f6c88b153e62ee85f0796)
Please take a look and confirm so that I create the other tests too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565873428
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
I am using this @the-other-tim-brown `mvn clean install -DskipTests -Dmaven.build.cache.enabled=false `
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2566024799
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ while (scanFiles.hasNext()) {
Review Comment:
I have updated the code, please check if that make sense. @the-other-tim-brown
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2433724274
##########
xtable-core/src/test/java/org/apache/xtable/delta/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.kernel.DeltaKernelConversionSource;
+import org.apache.xtable.kernel.DeltaKernelConversionSourceProvider;
+import org.apache.xtable.kernel.DeltaKernelPartitionExtractor;
+import org.apache.xtable.kernel.DeltaKernelSchemaExtractor;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.upsertRows(rows.subList(0, 20));
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(200L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testsShowingVacuumHasNoEffectOnIncrementalSync() {
+ boolean isPartitioned = true;
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // Insert 50 rows to 2018 partition.
+ List<Row> commit1Rows = testSparkDeltaTable.insertRowsForPartition(50, 2018);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ InternalSnapshot snapshotAfterCommit1 = conversionSource.getCurrentSnapshot();
+ List<String> allActivePaths = ValidationTestHelper.getAllFilePaths(snapshotAfterCommit1);
+ assertEquals(1, allActivePaths.size());
+ String activePathAfterCommit1 = allActivePaths.get(0);
+
+ // Upsert all rows inserted before, so all files are replaced.
+ testSparkDeltaTable.upsertRows(commit1Rows.subList(0, 50));
+
+ // Insert 50 rows to different (2020) partition.
+ testSparkDeltaTable.insertRowsForPartition(50, 2020);
+
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ conversionSource = conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ CommitsBacklog<Long> instantCurrentCommitState =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ assertTrue(conversionSource.isIncrementalSyncSafeFrom(Instant.ofEpochMilli(timestamp1)));
+ // // Table doesn't have instant of this older commit, hence it is not safe.
+ Instant instantAsOfHourAgo = Instant.now().minus(1, ChronoUnit.HOURS);
+ assertFalse(conversionSource.isIncrementalSyncSafeFrom(instantAsOfHourAgo));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testAddColumns(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, true);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(150L, testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+ if (isPartitioned) {
+ validateDeltaPartitioning(internalSnapshot);
+ }
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ public void testDropPartition() {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(tableName, tempDir, sparkSession, "yearOfBirth", false);
+ List<List<String>> allActiveFiles = new ArrayList<>();
+ List<TableChange> allTableChanges = new ArrayList<>();
+
+ List<Row> rows = testSparkDeltaTable.insertRows(50);
+ Long timestamp1 = testSparkDeltaTable.getLastCommitTimestamp();
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> rows1 = testSparkDeltaTable.insertRows(50);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ List<Row> allRows = new ArrayList<>();
+ allRows.addAll(rows);
+ allRows.addAll(rows1);
+
+ Map<Integer, List<Row>> rowsByPartition = testSparkDeltaTable.getRowsByPartition(allRows);
+ Integer partitionValueToDelete = rowsByPartition.keySet().stream().findFirst().get();
+ testSparkDeltaTable.deletePartition(partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ // Insert few records for deleted partition again to make it interesting.
+ testSparkDeltaTable.insertRowsForPartition(20, partitionValueToDelete);
+ allActiveFiles.add(testSparkDeltaTable.getAllActiveFiles());
+
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(testSparkDeltaTable.getTableName())
+ .basePath(testSparkDeltaTable.getBasePath())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ assertEquals(
+ 120 - rowsByPartition.get(partitionValueToDelete).size(), testSparkDeltaTable.getNumRows());
+ InternalSnapshot internalSnapshot = conversionSource.getCurrentSnapshot();
+
+ validateDeltaPartitioning(internalSnapshot);
+ ValidationTestHelper.validateSnapshot(
+ internalSnapshot, allActiveFiles.get(allActiveFiles.size() - 1));
+ // Get changes in incremental format.
+ InstantsForIncrementalSync instantsForIncrementalSync =
+ InstantsForIncrementalSync.builder()
+ .lastSyncInstant(Instant.ofEpochMilli(timestamp1))
+ .build();
+ CommitsBacklog<Long> commitsBacklog =
+ conversionSource.getCommitsBacklog(instantsForIncrementalSync);
+ for (Long version : commitsBacklog.getCommitsToProcess()) {
+ TableChange tableChange = conversionSource.getTableChangeForCommit(version);
+ allTableChanges.add(tableChange);
+ }
+ ValidationTestHelper.validateTableChanges(allActiveFiles, allTableChanges);
+ }
+
+ @Test
+ void testConvertFromDeltaPartitionSinglePartition() {
Review Comment:
We want to follow the convention of having unit tests separated for the units that are under test. You can look at what we do in TestDeltaPartitionExtractor and TestDeltaSchemaExtractor
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3369069670
@vaibhavk1992 can you write up a summary of next steps and blockers for this feature?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3369576029
@vaibhavk1992 I pushed some changes in the latest commit to extract the add and remove files per version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2330609804
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.*;
+
+import javax.swing.*;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<Action>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
Review Comment:
@vinishjail97 This is where we need the implementation to happen for response conversions from kernel
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176558236
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.util.*;
+
+import io.delta.kernel.types.DataType;
+import io.delta.kernel.types.IntegerType;
+import io.delta.kernel.types.StringType;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
+
+ private static final String DELTA_COLUMN_MAPPING_ID = "delta.columnMapping.id";
+ private static final DeltaKernelSchemaExtractor INSTANCE = new DeltaKernelSchemaExtractor();
+ private static final Map<InternalSchema.MetadataKey, Object>
+ DEFAULT_TIMESTAMP_PRECISION_METADATA =
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION, InternalSchema.MetadataValue.MICROS);
+
+ public static DeltaKernelSchemaExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalSchema toInternalSchema_v2(StructType structType) {
Review Comment:
I think we can just call this `toInternalSchema`, since its in its own distinct class right?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176573230
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+
+ System.out.println("Kernelschema: " + schema);
Review Comment:
[nit] well need to remove these in final verison of the pr.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.List;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+
+ System.out.println("Kernelschema: " + schema);
Review Comment:
[nit] well need to remove these in final version of the pr.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vinishjail97 (via GitHub)" <gi...@apache.org>.
vinishjail97 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2175322806
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
why comment this?
##########
xtable-core/src/test/resources/junit-platform.properties:
##########
@@ -14,6 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-junit.jupiter.execution.parallel.enabled=true
+junit.jupiter.execution.parallel.enabled=false
Review Comment:
I have run the tests locally by setting this to true and they pass, can we revert this config and see the GH build?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2330611430
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.time.Instant;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+
+import org.apache.xtable.delta.DeltaKernelTableExtractor;
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.*;
+import org.apache.xtable.spi.extractor.ConversionSource;
+
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
Review Comment:
all the methods have been added, just the commit backlog one is not fully resolved.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2217842319
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Scan;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(Snapshot deltaSnapshot, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot, InternalSchema schema, boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+
+ Scan myScan = snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles = myScan.getScanFiles(engine);
+ this.dataFilesIterator =
+ Collections
+ .emptyIterator(); // Initialize the dataFilesIterator by iterating over the scan files
+ while (scanFiles.hasNext()) {
+ FilteredColumnarBatch scanFileColumnarBatch = scanFiles.next();
+ CloseableIterator<Row> scanFileRows = scanFileColumnarBatch.getRows();
+ while (scanFileRows.hasNext()) {
+ Row scanFileRow = scanFileRows.next();
Review Comment:
In that same test linked above, they use the AddFile object which may be useful to you:
```
AddFile addFile = new AddFile(scanFileRow.getStruct(scanFileRow.getSchema().indexOf("add")));
addFile.getStatsJson();
```
It is important to note that the provided json parsing currently [does not extract the stats](https://github.com/delta-io/delta/blob/master/kernel/kernel-api/src/main/java/io/delta/kernel/internal/util/StatsUtils.java#L32) so we will still need to rely on our own [parsing](https://github.com/apache/incubator-xtable/blob/main/xtable-core/src/main/java/org/apache/xtable/delta/DeltaStatsExtractor.java#L195) of the string.
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.delta;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import org.apache.hadoop.conf.Configuration;
+
+import io.delta.kernel.Scan;
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(Snapshot deltaSnapshot, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot, InternalSchema schema, boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+ Configuration hadoopConf = new Configuration();
+ Engine engine = DefaultEngine.create(hadoopConf);
+
+ Scan myScan = snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles = myScan.getScanFiles(engine);
Review Comment:
```suggestion
ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
CloseableIterator<FilteredColumnarBatch> scanFiles = myScan.getScanFiles(engine, includeColumnStats);
```
In order to get the stats, we need to pass in another option that is only available on the ScanImpl. Check out their [tests](https://github.com/delta-io/delta/blob/master/kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala#L1040) for a sample
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176553002
##########
xtable-core/pom.xml:
##########
@@ -110,6 +110,19 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>4.0.0</version>
Review Comment:
Can you add a properly in the root pom called <delta.kernel.version>4.0.0</delta.kernel.version>, instead of using the hardcoded value?
Also curious how you ended up choosing delta kernel version, is there some specific version that needs to align with delta lake version we have in the repo?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2176479611
##########
pom.xml:
##########
@@ -53,7 +53,7 @@
<module>xtable-utilities</module>
<module>xtable-aws</module>
<module>xtable-hive-metastore</module>
- <module>xtable-service</module>
+<!-- <module>xtable-service</module>-->
Review Comment:
yes, xtable-service was giving build failures. As xtable-service is independent of the delta kernel changes. Can we just review the delta kernel changes as of now? I just want to get my changes validated once and anyhow the final version of delta kernel changes would have xtable-service module too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2410779270
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+ // The immutable map from Java to Scala is not working, need to
+
+ scala.collection.mutable.Map<String, String> scalaMap =
+ JavaConverters.mapAsScalaMap(partitionValues);
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ scala.collection.mutable.Map<String, String> scalaMap =
+ JavaConverters.mapAsScalaMap(partitionValues);
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(removeFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .build();
+ }
+
+ public FileFormat convertToFileFormat(String provider) {
+ if (provider.equals("parquet")) {
+ return FileFormat.APACHE_PARQUET;
+ } else if (provider.equals("orc")) {
+ return FileFormat.APACHE_ORC;
+ }
+ throw new NotSupportedException(
+ String.format("delta file format %s is not recognized", provider));
+ }
+
+ static String getFullPathToFile(String dataFilePath, Table table) {
+ Configuration hadoopConf = new Configuration();
+ Engine myEngine = DefaultEngine.create(hadoopConf);
+ String tableBasePath = table.getPath(myEngine);
+ ;
+ // String tableBasePath = snapshot.dataPath().toUri().toString();
Review Comment:
Can this be cleaned up?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
Review Comment:
Remove the comment `// List<String>`?
##########
xtable-core/src/main/java/org/apache/xtable/delta/DeltaSchemaExtractor.java:
##########
@@ -41,22 +37,10 @@
import org.apache.xtable.model.schema.InternalType;
import org.apache.xtable.schema.SchemaUtils;
-/**
- * Converts between Delta and InternalTable schemas. Some items to be aware of:
- *
- * <ul>
- * <li>Delta schemas are represented as Spark StructTypes which do not have enums so the enum
Review Comment:
@vaibhavk1992 let's remove the changes to this file. They don't seem necessary
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ System.err.println(
Review Comment:
Use logging instead of print lines
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
Review Comment:
Nitpick: Use camelCase for variable names
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ this.dataFilesIterator =
+ Collections
+ .emptyIterator(); // Initialize the dataFilesIterator by iterating over the scan files
Review Comment:
This assignment does not seem necessary since we assign at line 139
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2426991316
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.*;
+
+import io.delta.kernel.types.*;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaPartitionExtractor;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
Review Comment:
@the-other-tim-brown I have added one of the unit test `testConvertFromDeltaPartitionSinglePartition`. Please confirm if this is looking fine, I will add others too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2410467916
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.*;
+
+import io.delta.kernel.types.*;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaPartitionExtractor;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.schema.SchemaUtils;
+
+public class DeltaKernelSchemaExtractor {
Review Comment:
I have checked in the current running test cases wherever we are using getCurrentSnapshot we are calling this class and the methods in it. So the checks are already in place. @the-other-tim-brown
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,541 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.spark.sql.types.Metadata;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.delta.ScalaUtils;
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
Review Comment:
I have checked in the current running test cases wherever we are using getCurrentSnapshot and making InternalTable we are using this class and the methods in it. So the checks are already in place. @the-other-tim-brown
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.util.*;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Value;
+import lombok.extern.log4j.Log4j2;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.annotation.JsonAnySetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+
+import io.delta.kernel.internal.actions.AddFile;
+
+import org.apache.xtable.collectors.CustomCollectors;
+import org.apache.xtable.delta.DeltaValueConverter;
+import org.apache.xtable.model.exception.ParseException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.stat.Range;
+
+/**
+ * DeltaStatsExtractor extracts column stats and also responsible for their serialization leveraging
+ * {@link DeltaValueConverter}.
+ */
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelStatsExtractor {
Review Comment:
I have checked in the current running test cases under test testInsertsUpsertsAndDeletes we are using getTableChangeForCommit so we are using this class and the methods in it. So the checks are already in place. @the-other-tim-brown
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2581421618
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final CloseableIterator<FilteredColumnarBatch> scanFiles;
+ private final FileFormat fileFormat;
+ private final Table table;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private final boolean includeColumnStats;
+
+ private CloseableIterator<Row> currentFileRows;
+ private InternalDataFile nextFile;
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ this.includeColumnStats = includeColumnStats;
+ this.table = table;
+ this.fields = schema.getFields();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ this.scanFiles = myScan.getScanFiles(engine, includeColumnStats);
+
+ // Initialize first element
+ this.nextFile = computeNext();
+ }
+
+ @Override
+ public void close() throws Exception {
+ try {
+ if (currentFileRows != null) {
+ currentFileRows.close();
+ }
+ } finally {
+ scanFiles.close();
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ return nextFile != null;
+ }
+
+ @Override
+ public InternalDataFile next() {
+ InternalDataFile current = nextFile;
+ nextFile = computeNext();
+ return current;
+ }
+
+ private InternalDataFile computeNext() {
+ try {
+ while (true) {
Review Comment:
The `while (true)` makes me a bit worried for a potential infinite loop. Can we change this to `while (currentFileRows != null || scanFiles.hasNext())`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559449964
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559690838
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
+ Set<DeltaLogActionUtils.DeltaAction> actionSet = new HashSet<>();
+ actionSet.add(DeltaLogActionUtils.DeltaAction.ADD);
+ actionSet.add(DeltaLogActionUtils.DeltaAction.REMOVE);
+ TableImpl tableImpl = (TableImpl) Table.forPath(engine, table.getPath(engine));
+
+ // getChanges returns CloseableIterator<ColumnarBatch>
+ try (CloseableIterator<ColumnarBatch> iter =
+ tableImpl.getChanges(engine, versionToStartFrom, endVersion, actionSet)) {
+ while (iter.hasNext()) {
+ ColumnarBatch batch = iter.next();
+ int addFileIndex = batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.ADD.colName);
+ int removeFileIndex =
+ batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.REMOVE.colName);
+
+ try (CloseableIterator<Row> rows = batch.getRows()) {
+ while (rows.hasNext()) {
+ Row row = rows.next();
+
+ // Get version (first column)
+ long version = row.getLong(0);
+ List<RowBackedAction> actions =
+ incrementalChangesByVersion.computeIfAbsent(version, k -> new ArrayList<>());
+
+ if (!row.isNullAt(addFileIndex)) {
+ Row addFile = row.getStruct(addFileIndex);
+ AddFile addAction = new AddFile(addFile);
+ actions.add(addAction);
+ }
+ if (!row.isNullAt(removeFileIndex)) {
+ Row removeFile = row.getStruct(removeFileIndex);
+ RemoveFile removeAction = new RemoveFile(removeFile);
+ actions.add(removeAction);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error reading kernel changes", e);
Review Comment:
Ok, added the read exception
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
Review Comment:
changes has been done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559457427
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
+ Set<DeltaLogActionUtils.DeltaAction> actionSet = new HashSet<>();
+ actionSet.add(DeltaLogActionUtils.DeltaAction.ADD);
+ actionSet.add(DeltaLogActionUtils.DeltaAction.REMOVE);
+ TableImpl tableImpl = (TableImpl) Table.forPath(engine, table.getPath(engine));
+
+ // getChanges returns CloseableIterator<ColumnarBatch>
+ try (CloseableIterator<ColumnarBatch> iter =
+ tableImpl.getChanges(engine, versionToStartFrom, endVersion, actionSet)) {
+ while (iter.hasNext()) {
+ ColumnarBatch batch = iter.next();
+ int addFileIndex = batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.ADD.colName);
+ int removeFileIndex =
+ batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.REMOVE.colName);
+
+ try (CloseableIterator<Row> rows = batch.getRows()) {
+ while (rows.hasNext()) {
+ Row row = rows.next();
+
+ // Get version (first column)
+ long version = row.getLong(0);
+ List<RowBackedAction> actions =
+ incrementalChangesByVersion.computeIfAbsent(version, k -> new ArrayList<>());
+
+ if (!row.isNullAt(addFileIndex)) {
+ Row addFile = row.getStruct(addFileIndex);
+ AddFile addAction = new AddFile(addFile);
+ actions.add(addAction);
+ }
+ if (!row.isNullAt(removeFileIndex)) {
+ Row removeFile = row.getStruct(removeFileIndex);
+ RemoveFile removeAction = new RemoveFile(removeFile);
+ actions.add(removeAction);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error reading kernel changes", e);
+ }
+ }
+
+ /**
+ * Returns the versions in sorted order. The start version is the next one after the last sync
+ * version to the target. The end version is the latest version in the Delta table at the time of
+ * initialization.
+ *
+ * @return
+ */
+ public List<Long> getVersionsInSortedOrder() {
+ List<Long> versions = new ArrayList<>(incrementalChangesByVersion.keySet());
+ versions.sort(Long::compareTo);
+ return versions;
+ }
+
+ public List<RowBackedAction> getActionsForVersion(Long version) {
+ Preconditions.checkArgument(
+ incrementalChangesByVersion.containsKey(version),
+ String.format("Version %s not found in the DeltaIncrementalChangesState.", version));
+ return incrementalChangesByVersion.get(version);
+ }
+
+ private List<Tuple2<Long, List<RowBackedAction>>> getChangesList(
+ scala.collection.Iterator<Tuple2<Object, Seq<RowBackedAction>>> scalaIterator) {
+ List<Tuple2<Long, List<RowBackedAction>>> changesList = new ArrayList<>();
+ Iterator<Tuple2<Object, Seq<RowBackedAction>>> javaIterator =
+ JavaConverters.asJavaIteratorConverter(scalaIterator).asJava();
+ while (javaIterator.hasNext()) {
+ Tuple2<Object, Seq<RowBackedAction>> currentChange = javaIterator.next();
+ changesList.add(
+ new Tuple2<>(
+ (Long) currentChange._1(),
+ JavaConverters.seqAsJavaListConverter(currentChange._2()).asJava()));
+ }
+ return changesList;
+ }
Review Comment:
ok, will remove.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2559821010
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ while (scanFiles.hasNext()) {
Review Comment:
@the-other-tim-brown
the scanFiles.hasNext() Returns true if the iteration has more elements. (In other words, returns true if next would return an element rather than throwing an exception.). I am not sure what do you mean by materializing here and what we have to do exactly.
I have followed this convention from the suggestion by the doc form kernel community. Please check this.
https://github.com/delta-io/delta/blob/master/kernel/USER_GUIDE.md
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565533162
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ while (scanFiles.hasNext()) {
Review Comment:
The idea is that the `DeltaDataFileIterator` is an iterator so we don't need to compute all the values in the constructor. We can instead compute the next value when `next` is called.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565539830
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
Is the Schema extraction the main part that will throw in this? Right now we're still catching generic exception instead of particular exceptions which is a bit of a code-smell and makes it difficult to understand what types of exceptions we expect here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2565878790
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
And I am using java 11
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "rahil-c (via GitHub)" <gi...@apache.org>.
rahil-c commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2566548729
##########
pom.xml:
##########
@@ -615,6 +616,20 @@
<artifactId>jettison</artifactId>
<version>1.5.4</version>
</dependency>
+ <dependency>
+ <groupId>io.delta</groupId>
+ <artifactId>delta-kernel-api</artifactId>
+ <version>${delta.kernel.version}</version>
+ <scope>provided</scope>
Review Comment:
@vaibhavk1992 @the-other-tim-brown When i tried reproducing by setting this to `compile` I hit the following errors, in xtable service. I am not sure why this breaks from having delta kernel as compile scope.
```
[ERROR] COMPILATION ERROR :
[INFO] -------------------------------------------------------------
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[35,30] package org.apache.hadoop.conf does not exist
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[75,17] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[105,11] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[140,72] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
[ERROR] /Users/rahil/workplace/incubator-xtable/xtable-service/src/main/java/org/apache/xtable/service/ConversionService.java:[175,7] cannot find symbol
symbol: class Configuration
location: class org.apache.xtable.service.ConversionService
```
This class seems to come from `hadoop-common`
<img width="313" height="233" alt="Screenshot 2025-11-26 at 1 35 32 PM" src="https://github.com/user-attachments/assets/fb780784-cf28-42b9-b924-10aa87d63469" />
In order to get past the build exception I included in the `xtable-service` pom and was able to get build to pass.
```
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</dependency>
```
I am ok with the exclusion idea that @the-other-tim-brown mentioned if it allows the build to pass. but wanted to know if this was more ideal instead?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "vaibhavk1992 (via GitHub)" <gi...@apache.org>.
vaibhavk1992 commented on PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#issuecomment-3548611874
@the-other-tim-brown @vinishjail97 @rahil-c All the comments have been addressed and the build is passing too.
Please review for the final merge.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2540283029
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+ // The immutable map from Java to Scala is not working, need to
Review Comment:
This comment seems incomplete, is there something we need to fix on the scala/java conversion?
##########
xtable-core/src/test/java/org/apache/xtable/DeltaTableKernel.java:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable;
+
+// import org.junit.jupiter.api.Test;
+//
+import static io.delta.kernel.internal.util.Utils.singletonCloseableIterator;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.junit.jupiter.api.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.delta.kernel.*;
+import io.delta.kernel.data.ColumnVector;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.defaults.*;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.data.ScanStateRow;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+import io.delta.kernel.utils.FileStatus;
+
+public class DeltaTableKernel {
+ private static final Logger logger = LoggerFactory.getLogger(DeltaTableKernel.class);
+
+ @Test
+ public void readDeltaKernel() throws IOException {
+ String myTablePath =
+ "/Users/vaibhakumar/Desktop/opensource/iceberg/warehouse/demo/nyc/taxis"; // fully qualified
Review Comment:
This looks like some local testing. Do we still need this? If so, is there a way to make it run with a table created in the TempDir?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.testutil.ColumnStatMapUtil.getColumnStats;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+import java.util.*;
+
+import org.junit.jupiter.api.Test;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import io.delta.kernel.expressions.Column;
+import io.delta.kernel.expressions.Literal;
+import io.delta.kernel.types.StringType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.testutil.ColumnStatMapUtil;
+
+public class TestDeltaKernelStatsExtractor {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ public void testDeltaStats() throws JsonProcessingException {
+ InternalSchema schema = ColumnStatMapUtil.getSchema();
+
+ List<ColumnStat> columnStats = getColumnStats();
+
+ String actualStats =
+ DeltaKernelStatsExtractor.getInstance().convertStatsToDeltaFormat(schema, 50L, columnStats);
+ Map<String, Object> actualStatsMap = MAPPER.readValue(actualStats, HashMap.class);
+ assertEquals(50, actualStatsMap.get("numRecords"));
+ Map<String, Object> minValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("minValues");
+ assertEquals(10, minValueStatsMap.get("long_field"));
+ assertEquals("a", minValueStatsMap.get("string_field"));
+ assertEquals(null, minValueStatsMap.get("null_string_field"));
+ assertEquals("2022-10-08 21:08:17", minValueStatsMap.get("timestamp_field"));
+ assertEquals("2022-10-08 21:08:17", minValueStatsMap.get("timestamp_micros_field"));
+ assertEquals(1.23, minValueStatsMap.get("float_field"));
+ assertEquals(1.23, minValueStatsMap.get("double_field"));
+ assertEquals(1.0, minValueStatsMap.get("decimal_field"));
+ // TOD0: Local timestamp depends on env where it is run, it is non determinstic and this has to
+ // be computed dynamically.
+ // assertEquals("2022-10-08 14:08:17", minValueStatsMap.get("local_timestamp_field"));
+ assertEquals("2019-10-12", minValueStatsMap.get("date_field"));
+ Map<String, Object> nestedMapInMinValueStatsMap =
+ (HashMap<String, Object>) minValueStatsMap.get("nested_struct_field");
+ assertEquals(500, nestedMapInMinValueStatsMap.get("nested_long_field"));
+
+ Map<String, Object> maxValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("maxValues");
+ assertEquals(20, maxValueStatsMap.get("long_field"));
+ assertEquals("c", maxValueStatsMap.get("string_field"));
+ assertEquals(null, maxValueStatsMap.get("null_string_field"));
+ assertEquals("2022-10-10 21:08:17", maxValueStatsMap.get("timestamp_field"));
+ assertEquals("2022-10-10 21:08:17", maxValueStatsMap.get("timestamp_micros_field"));
+ // TOD0: Local timestamp depends on env where it is run, it is non determinstic and this has to
+ // be computed dynamically.
+ // assertEquals("2022-10-10 14:08:17", maxValueStatsMap.get("local_timestamp_field"));
+ assertEquals("2020-10-12", maxValueStatsMap.get("date_field"));
+ assertEquals(6.54321, maxValueStatsMap.get("float_field"));
+ assertEquals(6.54321, maxValueStatsMap.get("double_field"));
+ assertEquals(2.0, maxValueStatsMap.get("decimal_field"));
+ Map<String, Object> nestedMapInMaxValueStatsMap =
+ (HashMap<String, Object>) maxValueStatsMap.get("nested_struct_field");
+ assertEquals(600, nestedMapInMaxValueStatsMap.get("nested_long_field"));
+
+ Map<String, Object> nullValueStatsMap =
+ (HashMap<String, Object>) actualStatsMap.get("nullCount");
+ assertEquals(4, nullValueStatsMap.get("long_field"));
+ assertEquals(1, nullValueStatsMap.get("string_field"));
+
+ assertEquals(3, nullValueStatsMap.get("null_string_field"));
+ assertEquals(105, nullValueStatsMap.get("timestamp_field"));
+ assertEquals(1, nullValueStatsMap.get("timestamp_micros_field"));
+ assertEquals(1, nullValueStatsMap.get("local_timestamp_field"));
+ assertEquals(250, nullValueStatsMap.get("date_field"));
+ assertEquals(2, nullValueStatsMap.get("float_field"));
+ assertEquals(3, nullValueStatsMap.get("double_field"));
+ assertEquals(1, nullValueStatsMap.get("decimal_field"));
+ Map<String, Object> nestedMapInNullCountMap =
+ (HashMap<String, Object>) nullValueStatsMap.get("nested_struct_field");
+ assertEquals(4, nestedMapInNullCountMap.get("nested_long_field"));
+ }
+
+ private Map<Column, Literal> parseValues(JsonNode valuesNode) {
+ Map<Column, Literal> values = new HashMap<>();
+ if (valuesNode == null || valuesNode.isNull()) {
+ return values;
+ }
+
+ Iterator<Map.Entry<String, JsonNode>> fields = valuesNode.fields();
+ while (fields.hasNext()) {
+ Map.Entry<String, JsonNode> entry = fields.next();
+ String columnName = entry.getKey();
+ JsonNode valueNode = entry.getValue();
+ values.put(new Column(columnName), convertToLiteral(valueNode));
+ }
+ return values;
+ }
+
+ private Literal convertToLiteral(JsonNode valueNode) {
+ System.out.println("ValueNode: " + valueNode);
Review Comment:
Remove this println
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ System.out.println("if coming");
Review Comment:
Remove the println here and below
##########
xtable-service/pom.xml:
##########
@@ -60,7 +60,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
</dependency>
-
+ <dependency>
Review Comment:
Why is this required as part of this change?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,538 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ System.out.println("if coming");
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ System.out.println("else coming");
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ scala.collection.Map<String, String> values, List<InternalPartitionField> partitionFields) {
Review Comment:
Can we just use a java map here?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/ITDeltaKernelConversionSource.java:
##########
@@ -0,0 +1,734 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.testutil.ITTestUtils.validateTable;
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.spark.serializer.KryoSerializer;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SparkSession;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.GenericTable;
+import org.apache.xtable.TestSparkDeltaTable;
+import org.apache.xtable.ValidationTestHelper;
+import org.apache.xtable.conversion.SourceTable;
+import org.apache.xtable.model.*;
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.*;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+public class ITDeltaKernelConversionSource {
+ private static final InternalField COL1_INT_FIELD =
+ InternalField.builder()
+ .name("col1")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL2_INT_FIELD =
+ InternalField.builder()
+ .name("col2")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+
+ private static final InternalField COL3_STR_FIELD =
+ InternalField.builder()
+ .name("col3")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ private static final ColumnStat COL2_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL2_INT_FIELD)
+ .range(Range.vector(2, 2))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+ private static final ColumnStat COL1_COLUMN_STAT =
+ ColumnStat.builder()
+ .field(COL1_INT_FIELD)
+ .range(Range.vector(1, 1))
+ .numNulls(0)
+ .numValues(1)
+ .totalSize(0)
+ .build();
+
+ private DeltaKernelConversionSourceProvider conversionSourceProvider;
+ private static SparkSession sparkSession;
+
+ @BeforeAll
+ public static void setupOnce() {
+ sparkSession =
+ SparkSession.builder()
+ .appName("TestDeltaTable")
+ .master("local[4]")
+ .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
+ .config(
+ "spark.sql.catalog.spark_catalog",
+ "org.apache.spark.sql.delta.catalog.DeltaCatalog")
+ .config("spark.databricks.delta.retentionDurationCheck.enabled", "false")
+ .config("spark.databricks.delta.schema.autoMerge.enabled", "true")
+ .config("spark.sql.shuffle.partitions", "1")
+ .config("spark.default.parallelism", "1")
+ .config("spark.serializer", KryoSerializer.class.getName())
+ .getOrCreate();
+ }
+
+ @TempDir private static Path tempDir;
+
+ @AfterAll
+ public static void tearDownSparkSession() {
+ if (sparkSession != null) {
+ sparkSession.catalog().clearCache();
+ sparkSession.stop();
+ }
+ }
+
+ @BeforeEach
+ void setUp() {
+ Configuration hadoopConf = new Configuration();
+ hadoopConf.set("fs.defaultFS", "file:///");
+
+ conversionSourceProvider = new DeltaKernelConversionSourceProvider();
+ conversionSourceProvider.init(hadoopConf);
+ }
+
+ @Test
+ void getCurrentSnapshotNonPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.emptyList());
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(Collections.emptyList())
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .partitionValues(Collections.emptyList())
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @Test
+ void getCurrentTableTest() {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ ;
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA LOCATION '"
+ + basePath
+ + "' AS SELECT * FROM VALUES (1, 2, 3)");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current table
+ InternalTable internalTable = conversionSource.getCurrentTable();
+ List<InternalField> fields = Arrays.asList(COL1_INT_FIELD, COL2_INT_FIELD, COL3_STR_FIELD);
+ validateTable(
+ internalTable,
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.FLAT,
+ "file://" + basePath,
+ internalTable.getLatestMetadataPath(),
+ Collections.emptyList());
+ }
+
+ @Test
+ void getCurrentSnapshotPartitionedTest() throws URISyntaxException {
+ // Table name
+ final String tableName = GenericTable.getTableName();
+ final Path basePath = tempDir.resolve(tableName);
+ // Create table with a single row using Spark
+ sparkSession.sql(
+ "CREATE TABLE `"
+ + tableName
+ + "` USING DELTA PARTITIONED BY (part_col)\n"
+ + "LOCATION '"
+ + basePath
+ + "' AS SELECT 'SingleValue' AS part_col, 1 AS col1, 2 AS col2");
+ // Create Delta source
+ SourceTable tableConfig =
+ SourceTable.builder()
+ .name(tableName)
+ .basePath(basePath.toString())
+ .formatName(TableFormat.DELTA)
+ .build();
+ DeltaKernelConversionSource conversionSource =
+ conversionSourceProvider.getConversionSourceInstance(tableConfig);
+ // Get current snapshot
+ InternalSnapshot snapshot = conversionSource.getCurrentSnapshot();
+ // Validate table
+ InternalField partCol =
+ InternalField.builder()
+ .name("part_col")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build();
+ List<InternalField> fields = Arrays.asList(partCol, COL1_INT_FIELD, COL2_INT_FIELD);
+ validateTable(
+ snapshot.getTable(),
+ tableName,
+ TableFormat.DELTA,
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .fields(fields)
+ .build(),
+ DataLayoutStrategy.HIVE_STYLE_PARTITION,
+ "file://" + basePath,
+ snapshot.getTable().getLatestMetadataPath(),
+ Collections.singletonList(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build()));
+ // Validate data files
+ List<ColumnStat> columnStats = Arrays.asList(COL1_COLUMN_STAT, COL2_COLUMN_STAT);
+ Assertions.assertEquals(1, snapshot.getPartitionedDataFiles().size());
+ List<PartitionValue> partitionValue =
+ Collections.singletonList(
+ PartitionValue.builder()
+ .partitionField(
+ InternalPartitionField.builder()
+ .sourceField(partCol)
+ .transformType(PartitionTransformType.VALUE)
+ .build())
+ .range(Range.scalar("SingleValue"))
+ .build());
+ validatePartitionDataFiles(
+ PartitionFileGroup.builder()
+ .partitionValues(partitionValue)
+ .files(
+ Collections.singletonList(
+ InternalDataFile.builder()
+ .physicalPath("file:/fake/path")
+ .fileFormat(FileFormat.APACHE_PARQUET)
+ .partitionValues(partitionValue)
+ .fileSizeBytes(716)
+ .recordCount(1)
+ .columnStats(columnStats)
+ .build()))
+ .build(),
+ snapshot.getPartitionedDataFiles().get(0));
+ }
+
+ @ParameterizedTest
+ @MethodSource("testWithPartitionToggle")
+ public void testInsertsUpsertsAndDeletes(boolean isPartitioned) {
+ String tableName = GenericTable.getTableName();
+ TestSparkDeltaTable testSparkDeltaTable =
+ new TestSparkDeltaTable(
+ tableName, tempDir, sparkSession, isPartitioned ? "yearOfBirth" : null, false);
+ // System.out.println("testSparkDeltaTable" + testSparkDeltaTable.getColumnsToSelect());
Review Comment:
Remove this?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelSchemaExtractor.java:
##########
@@ -0,0 +1,864 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.InternalType;
+
+public class TestDeltaKernelSchemaExtractor {
+ @Test
+ public void testPrimitiveTypes() {
+ Map<InternalSchema.MetadataKey, Object> decimalMetadata = new HashMap<>();
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_PRECISION, 10);
+ decimalMetadata.put(InternalSchema.MetadataKey.DECIMAL_SCALE, 2);
+
+ InternalSchema internalSchema =
+ InternalSchema.builder()
+ .name("struct")
+ .dataType(InternalType.RECORD)
+ .isNullable(false)
+ .fields(
+ Arrays.asList(
+ InternalField.builder()
+ .name("requiredBoolean")
+ .schema(
+ InternalSchema.builder()
+ .name("boolean")
+ .dataType(InternalType.BOOLEAN)
+ .isNullable(false)
+ .comment("requiredBooleanComment")
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalBoolean")
+ .schema(
+ InternalSchema.builder()
+ .name("boolean")
+ .dataType(InternalType.BOOLEAN)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredInt")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalInt")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredLong")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalLong")
+ .schema(
+ InternalSchema.builder()
+ .name("long")
+ .dataType(InternalType.LONG)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDouble")
+ .schema(
+ InternalSchema.builder()
+ .name("double")
+ .dataType(InternalType.DOUBLE)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDouble")
+ .schema(
+ InternalSchema.builder()
+ .name("double")
+ .dataType(InternalType.DOUBLE)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredFloat")
+ .schema(
+ InternalSchema.builder()
+ .name("float")
+ .dataType(InternalType.FLOAT)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalFloat")
+ .schema(
+ InternalSchema.builder()
+ .name("float")
+ .dataType(InternalType.FLOAT)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredString")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalString")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredBytes")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+ .dataType(InternalType.BYTES)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalBytes")
+ .schema(
+ InternalSchema.builder()
+ .name("binary")
+ .dataType(InternalType.BYTES)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDate")
+ .schema(
+ InternalSchema.builder()
+ .name("date")
+ .dataType(InternalType.DATE)
+ .isNullable(false)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDate")
+ .schema(
+ InternalSchema.builder()
+ .name("date")
+ .dataType(InternalType.DATE)
+ .isNullable(true)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build(),
+ InternalField.builder()
+ .name("requiredDecimal")
+ .schema(
+ InternalSchema.builder()
+ .name("decimal")
+ .dataType(InternalType.DECIMAL)
+ .isNullable(false)
+ .metadata(decimalMetadata)
+ .build())
+ .build(),
+ InternalField.builder()
+ .name("optionalDecimal")
+ .schema(
+ InternalSchema.builder()
+ .name("decimal")
+ .dataType(InternalType.DECIMAL)
+ .isNullable(true)
+ .metadata(decimalMetadata)
+ .build())
+ .defaultValue(InternalField.Constants.NULL_DEFAULT_VALUE)
+ .build()))
+ .build();
+ io.delta.kernel.types.StructType structRepresentation =
Review Comment:
Let's import the class instead of relying on the FQDN
##########
pom.xml:
##########
@@ -724,7 +725,7 @@
</executions>
<configuration>
<skip>${skipUTs}</skip>
- <redirectTestOutputToFile>true</redirectTestOutputToFile>
+ <redirectTestOutputToFile>false</redirectTestOutputToFile>
Review Comment:
Let's keep this as `true` for now?
##########
xtable-core/src/test/java/org/apache/xtable/kernel/TestDeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.kernel.DeltaKernelPartitionExtractor.DELTA_GENERATION_EXPRESSION;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+import org.junit.jupiter.api.Test;
+
+import scala.collection.JavaConverters;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.schema.*;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+
+public class TestDeltaKernelPartitionExtractor {
+ private static final Map<String, StructField> STRUCT_FIELD_MAP =
+ new HashMap<String, StructField>() {
+ {
+ put("id", new StructField("id", IntegerType.INTEGER, false));
+ put("firstName", new StructField("firstName", StringType.STRING, false));
+ put("gender", new StructField("gender", StringType.STRING, false));
+ put("birthDate", new StructField("birthDate", TimestampType.TIMESTAMP, false));
+ put(
+ "dateOfBirth",
+ new StructField(
+ "dateOfBirth",
+ DateType.DATE,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "CAST(birthDate AS DATE)")
+ .build()));
+
+ put(
+ "dateFmt",
+ new StructField(
+ "dateFmt",
+ StringType.STRING,
+ false,
+ FieldMetadata.builder()
+ .putString(
+ "delta.generationExpression", "DATE_FORMAT(birthDate, 'yyyy-MM-dd-HH')")
+ .build()));
+
+ put(
+ "yearOfBirth",
+ new StructField(
+ "yearOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "YEAR(birthDate)")
+ .build()));
+ put(
+ "monthOfBirth",
+ new StructField(
+ "monthOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "MONTH(birthDate)")
+ .build()));
+
+ put(
+ "dayOfBirth",
+ new StructField(
+ "dayOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "DAY(birthDate)")
+ .build()));
+
+ put(
+ "hourOfBirth",
+ new StructField(
+ "hourOfBirth",
+ IntegerType.INTEGER,
+ false,
+ FieldMetadata.builder()
+ .putString("delta.generationExpression", "HOUR(birthDate)")
+ .build()));
+ }
+ };
+ private static final InternalSchema TIMESTAMP_SCHEMA =
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .metadata(
+ Collections.singletonMap(
+ InternalSchema.MetadataKey.TIMESTAMP_PRECISION,
+ InternalSchema.MetadataValue.MICROS))
+ .build();
+ private final DeltaKernelPartitionExtractor deltaKernelPartitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+ private final DeltaKernelSchemaExtractor deltaKernelSchemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ @Test
+ public void testUnpartitionedTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, new StructType());
+ assertTrue(internalPartitionFields.isEmpty());
+ }
+
+ @Test
+ public void testSimplePartitionedTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("gender"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("gender")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDatePartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateOfBirth"));
+
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.DAY)
+ .partitionFieldNames(Collections.singletonList("dateOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDateFormatPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("dateFmt"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.HOUR)
+ .partitionFieldNames(Collections.singletonList("dateFmt"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.YEAR)
+ .partitionFieldNames(Collections.singletonList("yearOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearAndSimpleCombinedPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "yearOfBirth"));
+ StructType partitionSchema = getSchemaWithFields(Arrays.asList("yearOfBirth", "id"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.YEAR)
+ .partitionFieldNames(Collections.singletonList("yearOfBirth"))
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void yearMonthDayHourPartitionedGeneratedColumnsTable() {
+ StructType tableSchema =
+ getSchemaWithFields(
+ Arrays.asList(
+ "id",
+ "firstName",
+ "gender",
+ "birthDate",
+ "yearOfBirth",
+ "monthOfBirth",
+ "dayOfBirth",
+ "hourOfBirth"));
+ StructType partitionSchema =
+ getSchemaWithFields(
+ Arrays.asList("yearOfBirth", "monthOfBirth", "dayOfBirth", "hourOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .partitionFieldNames(
+ Arrays.asList("yearOfBirth", "monthOfBirth", "dayOfBirth", "hourOfBirth"))
+ .transformType(PartitionTransformType.HOUR)
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ // Test for preserving order of partition columns.
+ @Test
+ public void testCombinationOfPlainAndGeneratedColumns() {
+ StructType tableSchema =
+ getSchemaWithFields(Arrays.asList("id", "firstName", "gender", "birthDate", "dateFmt"));
+ StructType partitionSchema =
+ getSchemaWithFields(Arrays.asList("id", "dateFmt", "gender", "dateOfBirth"));
+ InternalSchema internalSchema = deltaKernelSchemaExtractor.toInternalSchema(tableSchema);
+ List<InternalPartitionField> expectedInternalPartitionFields =
+ Arrays.asList(
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("id")
+ .schema(
+ InternalSchema.builder()
+ .name("integer")
+ .dataType(InternalType.INT)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.HOUR)
+ .partitionFieldNames(Collections.singletonList("dateFmt"))
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("gender")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build(),
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder().name("birthDate").schema(TIMESTAMP_SCHEMA).build())
+ .transformType(PartitionTransformType.DAY)
+ .partitionFieldNames(Collections.singletonList("dateOfBirth"))
+ .build());
+ List<InternalPartitionField> internalPartitionFields =
+ deltaKernelPartitionExtractor.convertFromDeltaPartitionFormat(
+ internalSchema, partitionSchema);
+ assertEquals(expectedInternalPartitionFields, internalPartitionFields);
+ }
+
+ @Test
+ public void testDateFormatGeneratedPartitionValueExtraction() {
+ // date_partition_column is generated in the table as DATE_FORMAT(some_date_column,
+ // 'yyyy-MM-dd-HH')
+ // where some_date_column is of timestamp type.
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("date_partition_column", "2013-08-20-10");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(Collections.singletonList("date_partition_column"))
+ .transformType(PartitionTransformType.HOUR)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar(1376992800000L);
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ public void testSimplePartitionValueExtraction() {
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("partition_column2", "partition_value2");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column2")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar("partition_value2");
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ public void testYearMonthDayHourGeneratedPartitionValueExtraction() {
+ // year, month and day are generated in the table as based on some_date_column which is of
+ // timestamp type.
+ Map<String, String> partitionValuesMap =
+ new HashMap<String, String>() {
+ {
+ put("partition_column1", "partition_value1");
+ put("year_partition_column", "2013");
+ put("month_partition_column", "8");
+ put("day_partition_column", "20");
+ }
+ };
+ scala.collection.mutable.Map<String, String> scalaMap =
+ convertJavaMapToScalaMap(partitionValuesMap);
+ InternalPartitionField internalPartitionField1 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.VALUE)
+ .build();
+ InternalPartitionField internalPartitionField2 =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("some_date_column")
+ .schema(
+ InternalSchema.builder()
+ .name("timestamp")
+ .dataType(InternalType.TIMESTAMP)
+ .build())
+ .build())
+ .partitionFieldNames(
+ Arrays.asList(
+ "year_partition_column", "month_partition_column", "day_partition_column"))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ Range rangeForPartitionField1 = Range.scalar("partition_value1");
+ Range rangeForPartitionField2 = Range.scalar(1376956800000L);
+ List<PartitionValue> expectedPartitionValues =
+ Arrays.asList(
+ PartitionValue.builder()
+ .partitionField(internalPartitionField1)
+ .range(rangeForPartitionField1)
+ .build(),
+ PartitionValue.builder()
+ .partitionField(internalPartitionField2)
+ .range(rangeForPartitionField2)
+ .build());
+ List<PartitionValue> partitionValues =
+ deltaKernelPartitionExtractor.partitionValueExtraction(
+ scalaMap, Arrays.asList(internalPartitionField1, internalPartitionField2));
+ assertEquals(expectedPartitionValues, partitionValues);
+ }
+
+ @Test
+ void convertBucketPartition() {
+ InternalPartitionField internalPartitionField =
+ InternalPartitionField.builder()
+ .sourceField(
+ InternalField.builder()
+ .name("partition_column1")
+ .schema(
+ InternalSchema.builder()
+ .name("string")
+ .dataType(InternalType.STRING)
+ .build())
+ .build())
+ .transformType(PartitionTransformType.BUCKET)
+ .transformOptions(Collections.singletonMap(InternalPartitionField.NUM_BUCKETS, 5))
+ .build();
+ System.out.println("internalPartitionField" + internalPartitionField);
Review Comment:
Remove the println in this class as well
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] Delta Kernel Draft PR [incubator-xtable]
Posted by "the-other-tim-brown (via GitHub)" <gi...@apache.org>.
the-other-tim-brown commented on code in PR #729:
URL: https://github.com/apache/incubator-xtable/pull/729#discussion_r2557197815
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
Review Comment:
nitpick: remove star import
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
Review Comment:
```suggestion
List<String> partitionColumns = snapshot.getPartitionColumnNames();
```
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelConversionSource.java:
##########
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.sql.Timestamp;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import lombok.Builder;
+import lombok.extern.slf4j.Slf4j;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.internal.util.VectorUtils;
+
+import org.apache.xtable.exception.ReadException;
+import org.apache.xtable.model.CommitsBacklog;
+import org.apache.xtable.model.InstantsForIncrementalSync;
+import org.apache.xtable.model.InternalSnapshot;
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.TableChange;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.model.storage.InternalFilesDiff;
+import org.apache.xtable.model.storage.PartitionFileGroup;
+import org.apache.xtable.spi.extractor.ConversionSource;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+@Slf4j
+@Builder
+public class DeltaKernelConversionSource implements ConversionSource<Long> {
+
+ @Builder.Default
+ private final DeltaKernelDataFileExtractor dataFileExtractor =
+ DeltaKernelDataFileExtractor.builder().build();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+ private final String tableName;
+ private final Engine engine;
+
+ @Builder.Default
+ private final DeltaKernelTableExtractor tableExtractor =
+ DeltaKernelTableExtractor.builder().build();
+
+ private Optional<DeltaKernelIncrementalChangesState> deltaKernelIncrementalChangesState =
+ Optional.empty();
+
+ @Override
+ public InternalTable getTable(Long version) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, version);
+ return tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ } catch (Exception e) {
+ throw new ReadException("Failed to get table at version " + version, e);
+ }
+ }
+
+ @Override
+ public InternalTable getCurrentTable() {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getLatestSnapshot(engine);
+ return getTable(snapshot.getVersion());
+ }
+
+ @Override
+ public InternalSnapshot getCurrentSnapshot() {
+ Table table_snapshot = Table.forPath(engine, basePath);
+ Snapshot snapshot = table_snapshot.getLatestSnapshot(engine);
+ InternalTable table = getTable(snapshot.getVersion());
+ return InternalSnapshot.builder()
+ .table(table)
+ .partitionedDataFiles(
+ getInternalDataFiles(snapshot, table_snapshot, engine, table.getReadSchema()))
+ .sourceIdentifier(getCommitIdentifier(snapshot.getVersion()))
+ .build();
+ }
+
+ @Override
+ public TableChange getTableChangeForCommit(Long versionNumber) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfVersion(engine, versionNumber);
+ InternalTable tableAtVersion =
+ tableExtractor.table(table, snapshot, engine, tableName, basePath);
+ Map<String, InternalDataFile> addedFiles = new HashMap<>();
+ Map<String, InternalDataFile> removedFiles = new HashMap<>();
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ FileFormat fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ List<RowBackedAction> actionsForVersion = getChangesState().getActionsForVersion(versionNumber);
+
+ for (RowBackedAction action : actionsForVersion) {
+ if (action instanceof AddFile) {
+ AddFile addFile = (AddFile) action;
+ Map<String, String> partitionValues = VectorUtils.toJavaMap(addFile.getPartitionValues());
+ InternalDataFile dataFile =
+ actionsConverter.convertAddActionToInternalDataFile(
+ addFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ tableAtVersion.getReadSchema().getFields(),
+ true,
+ DeltaKernelPartitionExtractor.getInstance(),
+ DeltaKernelStatsExtractor.getInstance(),
+ partitionValues);
+ addedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ } else if (action instanceof RemoveFile) {
+ RemoveFile removeFile = (RemoveFile) action;
+ Map<String, String> partitionValues =
+ removeFile
+ .getPartitionValues()
+ .map(VectorUtils::<String, String>toJavaMap)
+ .orElse(Collections.emptyMap());
+ InternalDataFile dataFile =
+ actionsConverter.convertRemoveActionToInternalDataFile(
+ removeFile,
+ table,
+ fileFormat,
+ tableAtVersion.getPartitioningFields(),
+ DeltaKernelPartitionExtractor.getInstance(),
+ partitionValues);
+ removedFiles.put(dataFile.getPhysicalPath(), dataFile);
+ }
+ }
+
+ InternalFilesDiff internalFilesDiff =
+ InternalFilesDiff.builder()
+ .filesAdded(addedFiles.values())
+ .filesRemoved(removedFiles.values())
+ .build();
+ return TableChange.builder()
+ .tableAsOfChange(tableAtVersion)
+ .filesDiff(internalFilesDiff)
+ .sourceIdentifier(getCommitIdentifier(versionNumber))
+ .build();
+ }
+
+ @Override
+ public CommitsBacklog<Long> getCommitsBacklog(
+ InstantsForIncrementalSync instantsForIncrementalSync) {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot =
+ table.getSnapshotAsOfTimestamp(
+ engine, Timestamp.from(instantsForIncrementalSync.getLastSyncInstant()).getTime());
+
+ long versionNumberAtLastSyncInstant = snapshot.getVersion();
+ resetState(versionNumberAtLastSyncInstant + 1, engine, table);
+ return CommitsBacklog.<Long>builder()
+ .commitsToProcess(getChangesState().getVersionsInSortedOrder())
+ .build();
+ }
+
+ @Override
+ public boolean isIncrementalSyncSafeFrom(Instant instant) {
+ try {
+ Table table = Table.forPath(engine, basePath);
+ Snapshot snapshot = table.getSnapshotAsOfTimestamp(engine, Timestamp.from(instant).getTime());
+
+ // There is a chance earliest commit of the table is returned if the instant is before the
+ // earliest commit of the table, hence the additional check.
+ Instant deltaCommitInstant = Instant.ofEpochMilli(snapshot.getTimestamp(engine));
+ return deltaCommitInstant.equals(instant) || deltaCommitInstant.isBefore(instant);
+ } catch (Exception e) {
+ log.error(
+ "Error checking if incremental sync is safe from " + instant + ": " + e.getMessage());
Review Comment:
We'll want the full stacktrace for debugging
```suggestion
"Error checking if incremental sync is safe from " + instant, e);
```
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
Review Comment:
Just remove this instead of commenting it out
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
Review Comment:
```suggestion
long versionToStartFrom, Engine engine, Table table, long endVersion) {
```
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
+ Set<DeltaLogActionUtils.DeltaAction> actionSet = new HashSet<>();
+ actionSet.add(DeltaLogActionUtils.DeltaAction.ADD);
+ actionSet.add(DeltaLogActionUtils.DeltaAction.REMOVE);
+ TableImpl tableImpl = (TableImpl) Table.forPath(engine, table.getPath(engine));
+
+ // getChanges returns CloseableIterator<ColumnarBatch>
+ try (CloseableIterator<ColumnarBatch> iter =
+ tableImpl.getChanges(engine, versionToStartFrom, endVersion, actionSet)) {
+ while (iter.hasNext()) {
+ ColumnarBatch batch = iter.next();
+ int addFileIndex = batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.ADD.colName);
+ int removeFileIndex =
+ batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.REMOVE.colName);
+
+ try (CloseableIterator<Row> rows = batch.getRows()) {
+ while (rows.hasNext()) {
+ Row row = rows.next();
+
+ // Get version (first column)
+ long version = row.getLong(0);
+ List<RowBackedAction> actions =
+ incrementalChangesByVersion.computeIfAbsent(version, k -> new ArrayList<>());
+
+ if (!row.isNullAt(addFileIndex)) {
+ Row addFile = row.getStruct(addFileIndex);
+ AddFile addAction = new AddFile(addFile);
+ actions.add(addAction);
+ }
+ if (!row.isNullAt(removeFileIndex)) {
+ Row removeFile = row.getStruct(removeFileIndex);
+ RemoveFile removeAction = new RemoveFile(removeFile);
+ actions.add(removeAction);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error reading kernel changes", e);
Review Comment:
Can we catch and throw a more specific exception defined in the XTable repo here? Maybe ReadException?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelStatsExtractor.java:
##########
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.io.IOException;
+import java.util.*;
Review Comment:
Similarly, avoid`*` imports
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
Review Comment:
Let's follow the java naming conventions and remove the underscore from the name. `partitionFieldSchemas` may also be a more clear name
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
Review Comment:
Use the imported StructType instead of relying on the verbose, fully-qualified class name
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ java.util.Map<String, String> scalaMap = partitionValues;
+
+ return InternalDataFile.builder()
+ .physicalPath(getFullPathToFile(addFile.getPath(), table))
+ .fileFormat(fileFormat)
+ .fileSizeBytes(addFile.getSize())
+ .lastModified(addFile.getModificationTime())
+ .partitionValues(partitionExtractor.partitionValueExtraction(scalaMap, partitionFields))
+ .columnStats(columnStats)
+ .recordCount(recordCount)
+ .build();
+ }
+
+ public InternalDataFile convertRemoveActionToInternalDataFile(
+ RemoveFile removeFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ Map<String, String> partitionValues) {
+ java.util.Map<String, String> scalaMap = partitionValues;
Review Comment:
Same here, let's import Map instead of using fully qualified class names
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames(); // List<String>
+ List<StructField> partitionFields_strfld =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+ StructType partitionSchema = new StructType(partitionFields_strfld);
+
+ List<InternalPartitionField> partitionFields =
+ DeltaKernelPartitionExtractor.getInstance()
+ .convertFromDeltaPartitionFormat(internalSchema, partitionSchema);
+
+ DataLayoutStrategy dataLayoutStrategy =
+ !partitionFields.isEmpty()
+ ? DataLayoutStrategy.HIVE_STYLE_PARTITION
+ : DataLayoutStrategy.FLAT;
+
+ // Get the timestamp
+ long timestamp = snapshot.getTimestamp(engine);
+ return InternalTable.builder()
+ .tableFormat(TableFormat.DELTA)
+ .basePath(basePath)
+ .name(tableName)
+ .layoutStrategy(dataLayoutStrategy)
+ .partitioningFields(partitionFields)
+ .readSchema(internalSchema)
+ .latestCommitTime(Instant.ofEpochMilli(timestamp))
+ .latestMetadataPath(basePath + "/_delta_log")
+ .build();
+ } catch (Exception e) {
Review Comment:
Are there more specific exceptions we can catch here? Let's also throw a more specific exception than the generic RuntimeException
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelActionsConverter.java:
##########
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import lombok.AccessLevel;
+import lombok.NoArgsConstructor;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.defaults.engine.DefaultEngine;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+
+import org.apache.xtable.exception.NotSupportedException;
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.stat.ColumnStat;
+import org.apache.xtable.model.stat.FileStats;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelActionsConverter {
+ private static final DeltaKernelActionsConverter INSTANCE = new DeltaKernelActionsConverter();
+
+ public static DeltaKernelActionsConverter getInstance() {
+ return INSTANCE;
+ }
+
+ public InternalDataFile convertAddActionToInternalDataFile(
+ AddFile addFile,
+ Table table,
+ FileFormat fileFormat,
+ List<InternalPartitionField> partitionFields,
+ List<InternalField> fields,
+ boolean includeColumnStats,
+ DeltaKernelPartitionExtractor partitionExtractor,
+ DeltaKernelStatsExtractor fileStatsExtractor,
+ Map<String, String> partitionValues) {
+ FileStats fileStats = fileStatsExtractor.getColumnStatsForFile(addFile, fields);
+ List<ColumnStat> columnStats =
+ includeColumnStats ? fileStats.getColumnStats() : Collections.emptyList();
+ long recordCount = fileStats.getNumRecords();
+
+ java.util.Map<String, String> scalaMap = partitionValues;
Review Comment:
Add an import here for Map
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
Review Comment:
Avoid `*` import
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
Review Comment:
It looks like this is always set below so let's make this `final` and remove the `Collections.emptyIterator()` here
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelIncrementalChangesState.java:
##########
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import lombok.Builder;
+
+import scala.Tuple2;
+import scala.collection.JavaConverters;
+import scala.collection.Seq;
+
+import com.google.common.base.Preconditions;
+
+import io.delta.kernel.Table;
+import io.delta.kernel.data.ColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.DeltaLogActionUtils;
+import io.delta.kernel.internal.TableImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.internal.actions.RemoveFile;
+import io.delta.kernel.internal.actions.RowBackedAction;
+import io.delta.kernel.utils.CloseableIterator;
+
+/** Cache store for storing incremental table changes in the Delta table. */
+public class DeltaKernelIncrementalChangesState {
+
+ private final Map<Long, List<RowBackedAction>> incrementalChangesByVersion = new HashMap<>();
+
+ /**
+ * Reloads the cache store with incremental changes. Intentionally thread safety is the
+ * responsibility of the caller.
+ *
+ * @param engine The kernel engine.
+ * @param versionToStartFrom The version to start from.
+ */
+ @Builder
+ public DeltaKernelIncrementalChangesState(
+ Long versionToStartFrom, Engine engine, Table table, Long endVersion) {
+ Set<DeltaLogActionUtils.DeltaAction> actionSet = new HashSet<>();
+ actionSet.add(DeltaLogActionUtils.DeltaAction.ADD);
+ actionSet.add(DeltaLogActionUtils.DeltaAction.REMOVE);
+ TableImpl tableImpl = (TableImpl) Table.forPath(engine, table.getPath(engine));
+
+ // getChanges returns CloseableIterator<ColumnarBatch>
+ try (CloseableIterator<ColumnarBatch> iter =
+ tableImpl.getChanges(engine, versionToStartFrom, endVersion, actionSet)) {
+ while (iter.hasNext()) {
+ ColumnarBatch batch = iter.next();
+ int addFileIndex = batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.ADD.colName);
+ int removeFileIndex =
+ batch.getSchema().indexOf(DeltaLogActionUtils.DeltaAction.REMOVE.colName);
+
+ try (CloseableIterator<Row> rows = batch.getRows()) {
+ while (rows.hasNext()) {
+ Row row = rows.next();
+
+ // Get version (first column)
+ long version = row.getLong(0);
+ List<RowBackedAction> actions =
+ incrementalChangesByVersion.computeIfAbsent(version, k -> new ArrayList<>());
+
+ if (!row.isNullAt(addFileIndex)) {
+ Row addFile = row.getStruct(addFileIndex);
+ AddFile addAction = new AddFile(addFile);
+ actions.add(addAction);
+ }
+ if (!row.isNullAt(removeFileIndex)) {
+ Row removeFile = row.getStruct(removeFileIndex);
+ RemoveFile removeAction = new RemoveFile(removeFile);
+ actions.add(removeAction);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ throw new RuntimeException("Error reading kernel changes", e);
+ }
+ }
+
+ /**
+ * Returns the versions in sorted order. The start version is the next one after the last sync
+ * version to the target. The end version is the latest version in the Delta table at the time of
+ * initialization.
+ *
+ * @return
+ */
+ public List<Long> getVersionsInSortedOrder() {
+ List<Long> versions = new ArrayList<>(incrementalChangesByVersion.keySet());
+ versions.sort(Long::compareTo);
+ return versions;
+ }
+
+ public List<RowBackedAction> getActionsForVersion(Long version) {
+ Preconditions.checkArgument(
+ incrementalChangesByVersion.containsKey(version),
+ String.format("Version %s not found in the DeltaIncrementalChangesState.", version));
+ return incrementalChangesByVersion.get(version);
+ }
+
+ private List<Tuple2<Long, List<RowBackedAction>>> getChangesList(
+ scala.collection.Iterator<Tuple2<Object, Seq<RowBackedAction>>> scalaIterator) {
+ List<Tuple2<Long, List<RowBackedAction>>> changesList = new ArrayList<>();
+ Iterator<Tuple2<Object, Seq<RowBackedAction>>> javaIterator =
+ JavaConverters.asJavaIteratorConverter(scalaIterator).asJava();
+ while (javaIterator.hasNext()) {
+ Tuple2<Object, Seq<RowBackedAction>> currentChange = javaIterator.next();
+ changesList.add(
+ new Tuple2<>(
+ (Long) currentChange._1(),
+ JavaConverters.seqAsJavaListConverter(currentChange._2()).asJava()));
+ }
+ return changesList;
+ }
Review Comment:
This code is showing up as unused, can it be removed?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelDataFileExtractor.java:
##########
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+// import scala.collection.Map;
+import java.util.*;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.Snapshot;
+import io.delta.kernel.Table;
+import io.delta.kernel.data.FilteredColumnarBatch;
+import io.delta.kernel.data.Row;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.internal.InternalScanFileUtils;
+import io.delta.kernel.internal.ScanImpl;
+import io.delta.kernel.internal.SnapshotImpl;
+import io.delta.kernel.internal.actions.AddFile;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+import io.delta.kernel.utils.CloseableIterator;
+
+import org.apache.xtable.model.schema.InternalField;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.FileFormat;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.spi.extractor.DataFileIterator;
+
+/** DeltaDataFileExtractor lets the consumer iterate over partitions. */
+@Builder
+public class DeltaKernelDataFileExtractor {
+
+ @Builder.Default
+ private final DeltaKernelPartitionExtractor partitionExtractor =
+ DeltaKernelPartitionExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelStatsExtractor fileStatsExtractor =
+ DeltaKernelStatsExtractor.getInstance();
+
+ @Builder.Default
+ private final DeltaKernelActionsConverter actionsConverter =
+ DeltaKernelActionsConverter.getInstance();
+
+ private final String basePath;
+
+ /**
+ * Initializes an iterator for Delta Lake files.
+ *
+ * @return Delta table file iterator
+ */
+ public DataFileIterator iterator(
+ Snapshot deltaSnapshot, Table table, Engine engine, InternalSchema schema) {
+ return new DeltaDataFileIterator(deltaSnapshot, table, engine, schema, true);
+ }
+
+ public class DeltaDataFileIterator implements DataFileIterator {
+ private final FileFormat fileFormat;
+ private final List<InternalField> fields;
+ private final List<InternalPartitionField> partitionFields;
+ private Iterator<InternalDataFile> dataFilesIterator = Collections.emptyIterator();
+
+ private DeltaDataFileIterator(
+ Snapshot snapshot,
+ Table table,
+ Engine engine,
+ InternalSchema schema,
+ boolean includeColumnStats) {
+ String provider = ((SnapshotImpl) snapshot).getMetadata().getFormat().getProvider();
+ this.fileFormat = actionsConverter.convertToFileFormat(provider);
+
+ this.fields = schema.getFields();
+
+ StructType fullSchema = snapshot.getSchema(); // The full table schema
+ List<String> partitionColumns = snapshot.getPartitionColumnNames();
+
+ List<StructField> partitionFieldsStr =
+ fullSchema.fields().stream()
+ .filter(field -> partitionColumns.contains(field.getName()))
+ .collect(Collectors.toList());
+
+ StructType partitionSchema = new StructType(partitionFieldsStr);
+
+ this.partitionFields =
+ partitionExtractor.convertFromDeltaPartitionFormat(schema, partitionSchema);
+
+ ScanImpl myScan = (ScanImpl) snapshot.getScanBuilder().build();
+ CloseableIterator<FilteredColumnarBatch> scanFiles =
+ myScan.getScanFiles(engine, includeColumnStats);
+
+ List<InternalDataFile> dataFiles = new ArrayList<>();
+ while (scanFiles.hasNext()) {
Review Comment:
This is eagerly pulling all the files into memory instead of using the iterator pattern. Can the code be updated to iterate through these values when `next` is called instead of eagerly materializing them?
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelTableExtractor.java:
##########
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import lombok.Builder;
+
+import io.delta.kernel.*;
+import io.delta.kernel.engine.Engine;
+import io.delta.kernel.types.StructField;
+import io.delta.kernel.types.StructType;
+
+import org.apache.xtable.model.InternalTable;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.storage.DataLayoutStrategy;
+import org.apache.xtable.model.storage.TableFormat;
+
+/**
+ * Extracts {@link InternalTable} canonical representation of a table at a point in time for Delta.
+ */
+@Builder
+public class DeltaKernelTableExtractor {
+ @Builder.Default
+ private static final DeltaKernelSchemaExtractor schemaExtractor =
+ DeltaKernelSchemaExtractor.getInstance();
+
+ private final String basePath;
+
+ public InternalTable table(
+ Table deltaKernelTable, Snapshot snapshot, Engine engine, String tableName, String basePath) {
+ try {
+ // Get schema from Delta Kernel's snapshot
+ io.delta.kernel.types.StructType schema = snapshot.getSchema();
+ InternalSchema internalSchema = schemaExtractor.toInternalSchema(schema);
+ // Get partition columns);
Review Comment:
```suggestion
// Get partition columns
```
##########
xtable-core/src/main/java/org/apache/xtable/kernel/DeltaKernelPartitionExtractor.java:
##########
@@ -0,0 +1,535 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.xtable.kernel;
+
+import static org.apache.xtable.collectors.CustomCollectors.toList;
+import static org.apache.xtable.delta.DeltaValueConverter.convertFromDeltaPartitionValue;
+import static org.apache.xtable.delta.DeltaValueConverter.convertToDeltaPartitionValue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import lombok.AccessLevel;
+import lombok.Builder;
+import lombok.NoArgsConstructor;
+import lombok.extern.log4j.Log4j2;
+
+import scala.collection.JavaConverters;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.PeekingIterator;
+
+import io.delta.kernel.types.*;
+import io.delta.kernel.types.FieldMetadata;
+
+import org.apache.xtable.exception.PartitionSpecException;
+import org.apache.xtable.model.schema.InternalPartitionField;
+import org.apache.xtable.model.schema.InternalSchema;
+import org.apache.xtable.model.schema.PartitionTransformType;
+import org.apache.xtable.model.stat.PartitionValue;
+import org.apache.xtable.model.stat.Range;
+import org.apache.xtable.model.storage.InternalDataFile;
+import org.apache.xtable.schema.SchemaFieldFinder;
+
+@Log4j2
+@NoArgsConstructor(access = AccessLevel.PRIVATE)
+public class DeltaKernelPartitionExtractor {
+ private static final DeltaKernelPartitionExtractor INSTANCE = new DeltaKernelPartitionExtractor();
+ private static final String CAST_FUNCTION = "CAST(%s as DATE)";
+ private static final String DATE_FORMAT_FUNCTION = "DATE_FORMAT(%s, '%s')";
+ private static final String YEAR_FUNCTION = "YEAR(%s)";
+ private static final String DATE_FORMAT_FOR_HOUR = "yyyy-MM-dd-HH";
+ private static final String DATE_FORMAT_FOR_DAY = "yyyy-MM-dd";
+ private static final String DATE_FORMAT_FOR_MONTH = "yyyy-MM";
+ private static final String DATE_FORMAT_FOR_YEAR = "yyyy";
+ private static final String BUCKET_FUNCTION = "MOD((HASH(%s) & %d), %d)";
+ // For timestamp partition fields, actual partition column names in delta format will be of type
+ // generated & and with a name like `delta_partition_col_{transform_type}_{source_field_name}`.
+ private static final String DELTA_PARTITION_COL_NAME_FORMAT = "xtable_partition_col_%s_%s";
+ static final String DELTA_GENERATION_EXPRESSION = "delta.generationExpression";
+ private static final List<ParsedGeneratedExpr.GeneratedExprType> GRANULARITIES =
+ Arrays.asList(
+ ParsedGeneratedExpr.GeneratedExprType.YEAR,
+ ParsedGeneratedExpr.GeneratedExprType.MONTH,
+ ParsedGeneratedExpr.GeneratedExprType.DAY,
+ ParsedGeneratedExpr.GeneratedExprType.HOUR);
+
+ public static DeltaKernelPartitionExtractor getInstance() {
+ return INSTANCE;
+ }
+
+ /**
+ * Extracts partition fields from delta table. Partitioning by nested columns isn't supported.
+ * Example: Given a delta table and a reference to DeltaLog, method parameters can be obtained by
+ * deltaLog = DeltaLog.forTable(spark, deltaTablePath); InternalSchema internalSchema =
+ * DeltaSchemaExtractor.getInstance().toInternalSchema(deltaLog.snapshot().schema()); StructType
+ * partitionSchema = deltaLog.metadata().partitionSchema();
+ *
+ * @param internalSchema canonical representation of the schema.
+ * @param partitionSchema partition schema of the delta table.
+ * @return list of canonical representation of the partition fields
+ */
+ public List<InternalPartitionField> convertFromDeltaPartitionFormat(
+ InternalSchema internalSchema, StructType partitionSchema) {
+ if (partitionSchema.fields().size() == 0) {
+ return Collections.emptyList();
+ }
+ return getInternalPartitionFields(partitionSchema, internalSchema);
+ }
+
+ /**
+ * If all of them are value process individually and return. If they contain month they should
+ * contain year as well. If they contain day they should contain month and year as well. If they
+ * contain hour they should contain day, month and year as well. Other supports CAST(col as DATE)
+ * and DATE_FORMAT(col, 'yyyy-MM-dd'). Partition by nested fields may not be fully supported.
+ */
+ private List<InternalPartitionField> getInternalPartitionFields(
+ StructType partitionSchema, InternalSchema internalSchema) {
+ PeekingIterator<StructField> itr =
+ Iterators.peekingIterator(partitionSchema.fields().iterator());
+ List<InternalPartitionField> partitionFields = new ArrayList<>(partitionSchema.fields().size());
+ while (itr.hasNext()) {
+ StructField currPartitionField = itr.peek();
+ if (!currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ partitionFields.add(
+ InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, currPartitionField.getName()))
+ .transformType(PartitionTransformType.VALUE)
+ .build());
+ itr.next(); // consume the field.
+ } else {
+ // Partition contains generated expression.
+ // if it starts with year we should consume until we hit field with no generated expression
+ // or we hit a field with generated expression that is of cast or date format.
+ String expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ ParsedGeneratedExpr parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else if (ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ partitionFields.add(
+ getPartitionWithDateFormatTransform(
+ currPartitionField.getName(), parsedGeneratedExpr, internalSchema));
+ itr.next(); // consume the field.
+ } else {
+ // consume until we hit field with no generated expression or generated expression
+ // that is not of type cast or date format.
+ List<ParsedGeneratedExpr> parsedGeneratedExprs = new ArrayList<>();
+ while (itr.hasNext()
+ && currPartitionField.getMetadata().contains(DELTA_GENERATION_EXPRESSION)) {
+ expr = currPartitionField.getMetadata().getString(DELTA_GENERATION_EXPRESSION);
+ parsedGeneratedExpr =
+ ParsedGeneratedExpr.buildFromString(currPartitionField.getName(), expr);
+
+ if (ParsedGeneratedExpr.GeneratedExprType.CAST == parsedGeneratedExpr.generatedExprType
+ || ParsedGeneratedExpr.GeneratedExprType.DATE_FORMAT
+ == parsedGeneratedExpr.generatedExprType) {
+ break;
+ }
+ parsedGeneratedExprs.add(parsedGeneratedExpr);
+ itr.next(); // consume the field
+ if (itr.hasNext()) {
+ currPartitionField = itr.peek();
+ }
+ }
+ partitionFields.add(
+ getPartitionColumnsForHourOrDayOrMonthOrYear(parsedGeneratedExprs, internalSchema));
+ }
+ }
+ }
+ return partitionFields;
+ }
+
+ private InternalPartitionField getPartitionColumnsForHourOrDayOrMonthOrYear(
+ List<ParsedGeneratedExpr> parsedGeneratedExprs, InternalSchema internalSchema) {
+ if (parsedGeneratedExprs.size() > 4) {
+ throw new IllegalStateException("Invalid partition transform");
+ }
+ validate(
+ parsedGeneratedExprs, new HashSet<>(GRANULARITIES.subList(0, parsedGeneratedExprs.size())));
+
+ ParsedGeneratedExpr transform = parsedGeneratedExprs.get(0);
+ List<String> partitionColumns =
+ parsedGeneratedExprs.stream()
+ .map(parsedGeneratedExpr -> parsedGeneratedExpr.partitionColumnName)
+ .collect(toList(parsedGeneratedExprs.size()));
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance().findFieldByPath(internalSchema, transform.sourceColumn))
+ .partitionFieldNames(partitionColumns)
+ .transformType(
+ parsedGeneratedExprs.get(parsedGeneratedExprs.size() - 1)
+ .internalPartitionTransformType)
+ .build();
+ }
+
+ // Cast has default format of yyyy-MM-dd.
+ private InternalPartitionField getPartitionWithDateTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(PartitionTransformType.DAY)
+ .build();
+ }
+
+ private InternalPartitionField getPartitionWithDateFormatTransform(
+ String partitionColumnName,
+ ParsedGeneratedExpr parsedGeneratedExpr,
+ InternalSchema internalSchema) {
+ return InternalPartitionField.builder()
+ .sourceField(
+ SchemaFieldFinder.getInstance()
+ .findFieldByPath(internalSchema, parsedGeneratedExpr.sourceColumn))
+ .partitionFieldNames(Collections.singletonList(partitionColumnName))
+ .transformType(parsedGeneratedExpr.internalPartitionTransformType)
+ .build();
+ }
+
+ public Map<String, StructField> convertToDeltaPartitionFormat(
+ List<InternalPartitionField> partitionFields) {
+ if (partitionFields == null) {
+ return null;
+ }
+ Map<String, StructField> nameToStructFieldMap = new HashMap<>();
+ for (InternalPartitionField internalPartitionField : partitionFields) {
+ String currPartitionColumnName;
+ StructField field;
+
+ if (internalPartitionField.getTransformType() == PartitionTransformType.VALUE) {
+ currPartitionColumnName = internalPartitionField.getSourceField().getName();
+ field = null;
+ } else {
+ // Since partition field of timestamp or bucket type, create new field in schema.
+ field = getGeneratedField(internalPartitionField);
+ currPartitionColumnName = field.getName();
+ }
+ nameToStructFieldMap.put(currPartitionColumnName, field);
+ }
+ return nameToStructFieldMap;
+ }
+
+ public Map<String, String> partitionValueSerialization(InternalDataFile internalDataFile) {
+ Map<String, String> partitionValuesSerialized = new HashMap<>();
+ if (internalDataFile.getPartitionValues() == null
+ || internalDataFile.getPartitionValues().isEmpty()) {
+ return partitionValuesSerialized;
+ }
+ for (PartitionValue partitionValue : internalDataFile.getPartitionValues()) {
+ InternalPartitionField partitionField = partitionValue.getPartitionField();
+ PartitionTransformType transformType = partitionField.getTransformType();
+ String partitionValueSerialized;
+ if (transformType == PartitionTransformType.VALUE) {
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ "");
+ partitionValuesSerialized.put(
+ partitionField.getSourceField().getName(), partitionValueSerialized);
+ } else if (transformType == PartitionTransformType.BUCKET) {
+ partitionValueSerialized = partitionValue.getRange().getMaxValue().toString();
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ } else {
+ // use appropriate date formatter for value serialization.
+ partitionValueSerialized =
+ convertToDeltaPartitionValue(
+ partitionValue.getRange().getMaxValue(),
+ partitionField.getSourceField().getSchema().getDataType(),
+ transformType,
+ getDateFormat(partitionField.getTransformType()));
+ partitionValuesSerialized.put(
+ getGeneratedColumnName(partitionField), partitionValueSerialized);
+ }
+ }
+ return partitionValuesSerialized;
+ }
+
+ public List<PartitionValue> partitionValueExtraction(
+ java.util.Map<String, String> values, List<InternalPartitionField> partitionFields) {
Review Comment:
Let's add an import for Map in this class as well
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@xtable.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org