You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2023/03/22 07:22:31 UTC
[hudi] branch master updated: [MINOR] Fix typos in hudi-client and hudi-spark-datasource (#8230)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new e849ad828e5 [MINOR] Fix typos in hudi-client and hudi-spark-datasource (#8230)
e849ad828e5 is described below
commit e849ad828e5857d71147d69e7856213bda6a566b
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed Mar 22 00:22:24 2023 -0700
[MINOR] Fix typos in hudi-client and hudi-spark-datasource (#8230)
---
.../hudi/client/BaseHoodieTableServiceClient.java | 9 +++---
.../apache/hudi/client/BaseHoodieWriteClient.java | 32 ++++++++++----------
.../org/apache/hudi/config/HoodieLockConfig.java | 8 ++---
.../hudi/index/bloom/ListBasedIndexFileFilter.java | 2 +-
.../keygen/NonpartitionedAvroKeyGenerator.java | 5 ++--
.../metadata/HoodieBackedTableMetadataWriter.java | 6 ++--
.../apache/hudi/table/BulkInsertPartitioner.java | 2 +-
.../table/action/bootstrap/BootstrapUtils.java | 15 +++++-----
.../hudi/table/action/clean/CleanPlanner.java | 6 ++--
.../action/restore/BaseRestoreActionExecutor.java | 2 +-
.../table/action/rollback/BaseRollbackHelper.java | 7 +++--
.../rollback/BaseRollbackPlanActionExecutor.java | 2 +-
.../rollback/ListingBasedRollbackStrategy.java | 30 ++++++++++---------
.../SerializableHoodieRollbackRequest.java | 2 +-
.../hudi/table/upgrade/DowngradeHandler.java | 2 +-
...urrentFileWritesConflictResolutionStrategy.java | 20 +++++++------
.../factory/TestHoodieAvroKeyGeneratorFactory.java | 3 +-
.../io/storage/row/HoodieRowDataCreateHandle.java | 2 +-
.../hudi/io/storage/HoodieSparkFileReader.java | 2 +-
.../scala/org/apache/spark/sql/DataFrameUtil.scala | 2 +-
.../spark/sql/hudi/execution/RangeSample.scala | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 4 +--
.../hudi/client/TestHoodieClientMultiWriter.java | 2 +-
.../functional/TestHoodieBackedMetadata.java | 16 +++++-----
.../TestHoodieClientOnCopyOnWriteStorage.java | 12 ++++----
.../functional/TestHoodieMetadataBootstrap.java | 6 ++--
.../hudi/execution/TestSimpleExecutionInSpark.java | 5 ++--
.../index/hbase/TestSparkHoodieHBaseIndex.java | 22 +++++++-------
.../table/action/compact/TestInlineCompaction.java | 4 +--
.../strategy/TestHoodieCompactionStrategy.java | 2 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 4 +--
.../org/apache/hudi/ColumnStatsIndexSupport.scala | 2 +-
.../org/apache/hudi/HoodieBootstrapRelation.scala | 35 +++++++++++-----------
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 2 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 10 +++----
.../datasources/HoodieInMemoryFileIndex.scala | 4 +--
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 2 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 32 ++++++++++----------
.../hudi/command/payload/ExpressionPayload.scala | 7 ++---
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 16 ++++------
.../hudi/functional/TestMORDataSourceStorage.scala | 6 ++--
.../functional/cdc/TestCDCDataFrameSuite.scala | 4 +--
.../spark/sql/avro/TestSchemaConverters.scala | 2 +-
.../org/apache/spark/sql/hudi/TestSpark3DDL.scala | 9 +++---
44 files changed, 186 insertions(+), 183 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
index 5245c4bba21..79e7c816d10 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieTableServiceClient.java
@@ -207,7 +207,7 @@ public abstract class BaseHoodieTableServiceClient<O> extends BaseHoodieClient i
/***
* Schedules compaction inline.
- * @param extraMetadata extrametada to be used.
+ * @param extraMetadata extra metadata to be used.
* @return compaction instant if scheduled.
*/
protected Option<String> inlineScheduleCompaction(Option<Map<String, String>> extraMetadata) {
@@ -384,7 +384,7 @@ public abstract class BaseHoodieTableServiceClient<O> extends BaseHoodieClient i
*/
public Option<String> scheduleTableService(String instantTime, Option<Map<String, String>> extraMetadata,
TableServiceType tableServiceType) {
- // A lock is required to guard against race conditions between an on-going writer and scheduling a table service.
+ // A lock is required to guard against race conditions between an ongoing writer and scheduling a table service.
final Option<HoodieInstant> inflightInstant = Option.of(new HoodieInstant(HoodieInstant.State.REQUESTED,
tableServiceType.getAction(), instantTime));
try {
@@ -472,7 +472,7 @@ public abstract class BaseHoodieTableServiceClient<O> extends BaseHoodieClient i
/**
* Schedules clustering inline.
*
- * @param extraMetadata extrametadata to use.
+ * @param extraMetadata extra metadata to use.
* @return clustering instant if scheduled.
*/
protected Option<String> inlineScheduleClustering(Option<Map<String, String>> extraMetadata) {
@@ -584,7 +584,8 @@ public abstract class BaseHoodieTableServiceClient<O> extends BaseHoodieClient i
}
/**
- * Get inflight time line exclude compaction and clustering.
+ * Get inflight timeline excluding compaction and clustering.
+ *
* @param metaClient
* @return
*/
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 594d32de939..a8d6e72b0d8 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -110,7 +110,7 @@ import static org.apache.hudi.common.model.HoodieCommitMetadata.SCHEMA_KEY;
/**
* Abstract Write Client providing functionality for performing commit, index updates and rollback
- * Reused for regular write operations like upsert/insert/bulk-insert.. as well as bootstrap
+ * Reused for regular write operations like upsert/insert/bulk-insert as well as bootstrap
*
* @param <T> Type of data
* @param <I> Type of inputs
@@ -476,9 +476,9 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* Deletes a list of {@link HoodieKey}s from the Hoodie table, at the supplied instantTime {@link HoodieKey}s will be
- * de-duped and non existent keys will be removed before deleting.
+ * de-duped and non-existent keys will be removed before deleting.
*
- * @param keys {@link List} of {@link HoodieKey}s to be deleted
+ * @param keys {@link List} of {@link HoodieKey}s to be deleted
* @param instantTime Commit time handle
* @return Collection of WriteStatus to inspect errors and counts
*/
@@ -680,7 +680,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
* NOTE : This action requires all writers (ingest and compact) to a table to be stopped before proceeding. Revert
* the (inflight/committed) record changes for all commits after the provided instant time.
*
- * @param savepointToRestoreTimestamp savepoint intstant time to which restoration is requested
+ * @param savepointToRestoreTimestamp savepoint instant time to which restoration is requested
*/
public HoodieRestoreMetadata restoreToInstant(final String savepointToRestoreTimestamp, boolean initialMetadataTableIfNecessary) throws HoodieRestoreException {
LOG.info("Begin restore to instant " + savepointToRestoreTimestamp);
@@ -1030,7 +1030,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/***
* Schedules compaction inline.
- * @param extraMetadata extrametada to be used.
+ * @param extraMetadata extra metadata to be used.
* @return compaction instant if scheduled.
*/
protected Option<String> inlineScheduleCompaction(Option<Map<String, String>> extraMetadata) {
@@ -1305,10 +1305,10 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* add columns to table.
*
- * @param colName col name to be added. if we want to add col to a nested filed, the fullName should be specify
- * @param schema col type to be added.
- * @param doc col doc to be added.
- * @param position col position to be added
+ * @param colName col name to be added. if we want to add col to a nested filed, the fullName should be specified
+ * @param schema col type to be added.
+ * @param doc col doc to be added.
+ * @param position col position to be added
* @param positionType col position change type. now support three change types: first/after/before
*/
public void addColumn(String colName, Schema schema, String doc, String position, TableChange.ColumnPositionChange.ColumnPositionType positionType) {
@@ -1325,7 +1325,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* delete columns to table.
*
- * @param colNames col name to be deleted. if we want to delete col from a nested filed, the fullName should be specify
+ * @param colNames col name to be deleted. if we want to delete col from a nested filed, the fullName should be specified
*/
public void deleteColumns(String... colNames) {
Pair<InternalSchema, HoodieTableMetaClient> pair = getInternalSchemaAndMetaClient();
@@ -1336,7 +1336,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* rename col name for hudi table.
*
- * @param colName col name to be renamed. if we want to rename col from a nested filed, the fullName should be specify
+ * @param colName col name to be renamed. if we want to rename col from a nested filed, the fullName should be specified
* @param newName new name for current col. no need to specify fullName.
*/
public void renameColumn(String colName, String newName) {
@@ -1348,7 +1348,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* update col nullable attribute for hudi table.
*
- * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specify
+ * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specified
* @param nullable .
*/
public void updateColumnNullability(String colName, boolean nullable) {
@@ -1362,7 +1362,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
* only support update primitive type to primitive type.
* cannot update nest type to nest type or primitive type eg: RecordType -> MapType, MapType -> LongType.
*
- * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specify
+ * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specified
* @param newType .
*/
public void updateColumnType(String colName, Type newType) {
@@ -1374,8 +1374,8 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* update col comment for hudi table.
*
- * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specify
- * @param doc .
+ * @param colName col name to be changed. if we want to change col from a nested filed, the fullName should be specified
+ * @param doc .
*/
public void updateColumnComment(String colName, String doc) {
Pair<InternalSchema, HoodieTableMetaClient> pair = getInternalSchemaAndMetaClient();
@@ -1386,7 +1386,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O> extends BaseHoodieClient
/**
* reorder the position of col.
*
- * @param colName column which need to be reordered. if we want to change col from a nested filed, the fullName should be specify.
+ * @param colName column which need to be reordered. if we want to change col from a nested filed, the fullName should be specified.
* @param referColName reference position.
* @param orderType col position change type. now support three change types: first/after/before
*/
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
index 62aa6694275..f91c5791a03 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieLockConfig.java
@@ -112,10 +112,10 @@ public class HoodieLockConfig extends HoodieConfig {
.withDocumentation("For DFS based lock providers, path to store the locks under. use Table's meta path as default");
public static final ConfigProperty<Integer> FILESYSTEM_LOCK_EXPIRE = ConfigProperty
- .key(FILESYSTEM_LOCK_EXPIRE_PROP_KEY)
- .defaultValue(0)
- .sinceVersion("0.12.0")
- .withDocumentation("For DFS based lock providers, expire time in minutes, must be a nonnegative number, default means no expire");
+ .key(FILESYSTEM_LOCK_EXPIRE_PROP_KEY)
+ .defaultValue(0)
+ .sinceVersion("0.12.0")
+ .withDocumentation("For DFS based lock providers, expire time in minutes, must be a non-negative number, default means no expire");
public static final ConfigProperty<String> HIVE_DATABASE_NAME = ConfigProperty
.key(HIVE_DATABASE_NAME_PROP_KEY)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java
index c5e39146fbd..443166cf7b0 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/bloom/ListBasedIndexFileFilter.java
@@ -58,7 +58,7 @@ class ListBasedIndexFileFilter implements IndexFileFilter {
}
/**
- * if we dont have key ranges, then also we need to compare against the file. no other choice if we do, then only
+ * if we don't have key ranges, then also we need to compare against the file. no other choice if we do, then only
* compare the file if the record key falls in range.
*/
protected boolean shouldCompareWithFile(BloomIndexFileInfo indexInfo, String recordKey) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
index 5b5cedcbf88..59c883eaadc 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/keygen/NonpartitionedAvroKeyGenerator.java
@@ -17,17 +17,18 @@
package org.apache.hudi.keygen;
-import org.apache.avro.generic.GenericRecord;
import org.apache.hudi.common.config.TypedProperties;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
+import org.apache.avro.generic.GenericRecord;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
/**
- * Avro simple Key generator for unpartitioned Hive Tables.
+ * Avro simple Key generator for non-partitioned Hive Tables.
*/
public class NonpartitionedAvroKeyGenerator extends BaseKeyGenerator {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 36c51e39197..dd1e40cb972 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -162,7 +162,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
this.metadataWriteConfig = createMetadataWriteConfig(writeConfig, failedWritesCleaningPolicy);
enabled = true;
- // Inline compaction and auto clean is required as we dont expose this table outside
+ // Inline compaction and auto clean is required as we do not expose this table outside
ValidationUtils.checkArgument(!this.metadataWriteConfig.isAutoClean(),
"Cleaning is controlled internally for Metadata table.");
ValidationUtils.checkArgument(!this.metadataWriteConfig.inlineCompactionEnabled(),
@@ -1067,7 +1067,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
// metadata table.
final String compactionInstantTime = latestDeltaCommitTimeInMetadataTable + METADATA_COMPACTION_TIME_SUFFIX;
// we need to avoid checking compaction w/ same instant again.
- // lets say we trigger compaction after C5 in MDT and so compaction completes with C4001. but C5 crashed before completing in MDT.
+ // let's say we trigger compaction after C5 in MDT and so compaction completes with C4001. but C5 crashed before completing in MDT.
// and again w/ C6, we will re-attempt compaction at which point latest delta commit is C4 in MDT.
// and so we try compaction w/ instant C4001. So, we can avoid compaction if we already have compaction w/ same instant time.
if (!metadataMetaClient.getActiveTimeline().filterCompletedInstants().containsInstant(compactionInstantTime)
@@ -1112,7 +1112,7 @@ public abstract class HoodieBackedTableMetadataWriter implements HoodieTableMeta
Map<MetadataPartitionType, HoodieData<HoodieRecord>> partitionToRecordsMap = new HashMap<>();
- // skip file system listing to populate metadata records if its a fresh table.
+ // skip file system listing to populate metadata records if it's a fresh table.
// this is applicable only if the table already has N commits and metadata is enabled at a later point in time.
if (createInstantTime.equals(SOLO_COMMIT_TIMESTAMP)) { // SOLO_COMMIT_TIMESTAMP will be the initial commit time in MDT for a fresh table.
// If not, last completed commit in data table will be chosen as the initial commit time.
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BulkInsertPartitioner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BulkInsertPartitioner.java
index 844e9f4f0f8..4239e47234e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BulkInsertPartitioner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/BulkInsertPartitioner.java
@@ -53,7 +53,7 @@ public interface BulkInsertPartitioner<I> extends Serializable {
/**
* Return file group id prefix for the given data partition.
- * By defauult, return a new file group id prefix, so that incoming records will route to a fresh new file group
+ * By default, return a new file group id prefix, so that incoming records will route to a fresh new file group
*
* @param partitionId data partition
* @return
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
index 9b0db44ecf4..3e9e6b42a61 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/bootstrap/BootstrapUtils.java
@@ -18,6 +18,13 @@
package org.apache.hudi.table.action.bootstrap;
+import org.apache.hudi.avro.model.HoodieFileStatus;
+import org.apache.hudi.common.bootstrap.FileStatusUtils;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.fs.FSUtils;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.util.collection.Pair;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -25,12 +32,6 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hudi.avro.model.HoodieFileStatus;
-import org.apache.hudi.common.bootstrap.FileStatusUtils;
-import org.apache.hudi.common.engine.HoodieEngineContext;
-import org.apache.hudi.common.fs.FSUtils;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.util.collection.Pair;
import java.io.IOException;
import java.util.ArrayList;
@@ -125,7 +126,7 @@ public class BootstrapUtils {
}
private static PathFilter getExcludeMetaPathFilter() {
- // Avoid listing and including any folders under the metafolder
+ // Avoid listing and including any folders under the meta folder
return (path) -> !path.toString().contains(HoodieTableMetaClient.METAFOLDER_NAME);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
index 62fd5717c6c..6951746503d 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/clean/CleanPlanner.java
@@ -226,7 +226,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
/**
* Selects the older versions of files for cleaning, such that it bounds the number of versions of each file. This
* policy is useful, if you are simply interested in querying the table, and you don't want too many versions for a
- * single file (i.e run it with versionsRetained = 1)
+ * single file (i.e., run it with versionsRetained = 1)
*/
private Pair<Boolean, List<CleanFileInfo>> getFilesToCleanKeepingLatestVersions(String partitionPath) {
LOG.info("Cleaning " + partitionPath + ", retaining latest " + config.getCleanerFileVersionsRetained()
@@ -330,7 +330,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
getLatestVersionBeforeCommit(fileSliceList, earliestCommitToRetain);
// Ensure there are more than 1 version of the file (we only clean old files from updates)
- // i.e always spare the last commit.
+ // i.e., always spare the last commit.
for (FileSlice aSlice : fileSliceList) {
Option<HoodieBaseFile> aFile = aSlice.getBaseFile();
String fileCommitTime = aSlice.getBaseInstantTime();
@@ -340,7 +340,7 @@ public class CleanPlanner<T, I, K, O> implements Serializable {
}
if (policy == HoodieCleaningPolicy.KEEP_LATEST_COMMITS) {
- // Dont delete the latest commit and also the last commit before the earliest commit we
+ // Do not delete the latest commit and also the last commit before the earliest commit we
// are retaining
// The window of commit retain == max query run time. So a query could be running which
// still
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
index d74432e8aa7..bfcd49840b2 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/restore/BaseRestoreActionExecutor.java
@@ -103,7 +103,7 @@ public abstract class BaseRestoreActionExecutor<T, I, K, O> extends BaseActionEx
List<HoodieInstant> instantsToRollback = new ArrayList<>();
HoodieRestorePlan restorePlan = RestoreUtils.getRestorePlan(table.getMetaClient(), restoreInstant);
for (HoodieInstantInfo instantInfo : restorePlan.getInstantsToRollback()) {
- // If restore crashed mid-way, there are chances that some commits are already rolled back,
+ // If restore crashed midway, there are chances that some commits are already rolled back,
// but some are not. so, we can ignore those commits which are fully rolledback in previous attempt if any.
Option<HoodieInstant> rollbackInstantOpt = table.getActiveTimeline().getWriteTimeline()
.filter(instant -> instant.getTimestamp().equals(instantInfo.getCommitTime()) && instant.getAction().equals(instantInfo.getAction())).firstInstant();
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
index fa49d21391b..2a91bfcf4fe 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackHelper.java
@@ -18,8 +18,6 @@
package org.apache.hudi.table.action.rollback;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.Path;
import org.apache.hudi.avro.model.HoodieRollbackRequest;
import org.apache.hudi.common.HoodieRollbackStat;
import org.apache.hudi.common.engine.HoodieEngineContext;
@@ -35,6 +33,9 @@ import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieRollbackException;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -83,7 +84,7 @@ public class BaseRollbackHelper implements Serializable {
}
/**
- * Collect all file info that needs to be rollbacked.
+ * Collect all file info that needs to be rolled back.
*/
public List<HoodieRollbackStat> collectRollbackStats(HoodieEngineContext context, HoodieInstant instantToRollback,
List<HoodieRollbackRequest> rollbackRequests) {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackPlanActionExecutor.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackPlanActionExecutor.java
index c8aa992cf89..18525404866 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackPlanActionExecutor.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/BaseRollbackPlanActionExecutor.java
@@ -94,7 +94,7 @@ public class BaseRollbackPlanActionExecutor<T, I, K, O> extends BaseActionExecut
}
/**
- * Creates a Rollback plan if there are files to be rolledback and stores them in instant file.
+ * Creates a Rollback plan if there are files to be rolled back and stores them in instant file.
* Rollback Plan contains absolute file paths.
*
* @param startRollbackTime Rollback Instant Time
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
index 10b02d7fff2..86a971da036 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/ListingBasedRollbackStrategy.java
@@ -18,18 +18,6 @@
package org.apache.hudi.table.action.rollback;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.function.Function;
-import java.util.stream.Collectors;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hudi.avro.model.HoodieRollbackRequest;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.fs.FSUtils;
@@ -47,10 +35,24 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieRollbackException;
import org.apache.hudi.table.HoodieTable;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.jetbrains.annotations.NotNull;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
import static org.apache.hudi.client.utils.MetadataConversionUtils.getHoodieCommitMetadata;
import static org.apache.hudi.common.util.ValidationUtils.checkArgument;
import static org.apache.hudi.table.action.rollback.BaseRollbackHelper.EMPTY_STRING;
@@ -142,7 +144,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
// (A.2) Failed recurring commit - Inserts/Updates written to log files. In this scenario,
// HoodieWriteStat will have the baseCommitTime for the first log file written, add rollback blocks.
// (A.3) Rollback triggered for first commit - Inserts were written to the log files but the commit is
- // being reverted. In this scenario, HoodieWriteStat will be `null` for the attribute prevCommitTime and
+ // being reverted. In this scenario, HoodieWriteStat will be `null` for the attribute prevCommitTime
// and hence will end up deleting these log files. This is done so there are no orphan log files
// lying around.
// (A.4) Rollback triggered for recurring commits - Inserts/Updates are being rolled back, the actions
@@ -156,7 +158,7 @@ public class ListingBasedRollbackStrategy implements BaseRollbackPlanActionExecu
// this scenario, perform (A.1) and for updates written to log files, write rollback blocks.
// (B.3) Rollback triggered for first commit - Same as (B.1)
// (B.4) Rollback triggered for recurring commits - Same as (B.2) plus we need to delete the log files
- // as well if the base base file gets deleted.
+ // as well if the base file gets deleted.
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata.fromBytes(
table.getMetaClient().getCommitTimeline().getInstantDetails(instantToRollback).get(),
HoodieCommitMetadata.class);
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/SerializableHoodieRollbackRequest.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/SerializableHoodieRollbackRequest.java
index 8f19692ed7c..2d35638889e 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/SerializableHoodieRollbackRequest.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/rollback/SerializableHoodieRollbackRequest.java
@@ -27,7 +27,7 @@ import java.util.List;
import java.util.Map;
/**
- * HoodieRollbackRequest in HoodieRollbackPlan (avro pojo) is not operable direclty within spark parallel engine.
+ * HoodieRollbackRequest in HoodieRollbackPlan (avro pojo) is not operable directly within spark parallel engine.
* Hence converting the same to this {@link SerializableHoodieRollbackRequest} and then using it within spark.parallelize.
*/
public class SerializableHoodieRollbackRequest implements Serializable {
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java
index 45bbd78c3fb..51981e441de 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/DowngradeHandler.java
@@ -34,7 +34,7 @@ public interface DowngradeHandler {
*
* @param config instance of {@link HoodieWriteConfig} to be used.
* @param context instance of {@link HoodieEngineContext} to be used.
- * @param instantTime current instant time that should not touched.
+ * @param instantTime current instant time that should not be touched.
* @param upgradeDowngradeHelper instance of {@link SupportsUpgradeDowngrade} to be used.
* @return Map of config properties and its values to be added to table properties.
*/
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java
index 39b9e1e6dc4..e220c63f3e8 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategy.java
@@ -18,14 +18,6 @@
package org.apache.hudi.client.transaction;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import org.apache.hudi.avro.model.HoodieClusteringGroup;
import org.apache.hudi.avro.model.HoodieClusteringPlan;
import org.apache.hudi.avro.model.HoodieCompactionOperation;
@@ -48,10 +40,20 @@ import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.exception.HoodieWriteConflictException;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
public class TestSimpleConcurrentFileWritesConflictResolutionStrategy extends HoodieCommonTestHarness {
@BeforeEach
@@ -92,7 +94,7 @@ public class TestSimpleConcurrentFileWritesConflictResolutionStrategy extends Ho
}
@Test
- public void testConcurrentWritesWithInterleavingSuccesssfulCommit() throws Exception {
+ public void testConcurrentWritesWithInterleavingSuccessfulCommit() throws Exception {
createCommit(HoodieActiveTimeline.createNewInstantTime());
HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
// consider commits before this are all successful
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
index c3be6284520..88bd5091ccd 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/keygen/factory/TestHoodieAvroKeyGeneratorFactory.java
@@ -25,6 +25,7 @@ import org.apache.hudi.keygen.KeyGenerator;
import org.apache.hudi.keygen.SimpleAvroKeyGenerator;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
import org.apache.hudi.keygen.constant.KeyGeneratorType;
+
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -51,7 +52,7 @@ public class TestHoodieAvroKeyGeneratorFactory {
// set both class name and keyGenerator type
props.put(HoodieWriteConfig.KEYGENERATOR_TYPE.key(), KeyGeneratorType.CUSTOM.name());
KeyGenerator keyGenerator3 = HoodieAvroKeyGeneratorFactory.createKeyGenerator(props);
- // KEYGENERATOR_TYPE_PROP was overitten by KEYGENERATOR_CLASS_PROP
+ // KEYGENERATOR_TYPE_PROP was overwritten by KEYGENERATOR_CLASS_PROP
Assertions.assertEquals(SimpleAvroKeyGenerator.class.getName(), keyGenerator3.getClass().getName());
// set wrong class name
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index 231dee2c7d4..3a5f2105451 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -47,7 +47,7 @@ import java.io.Serializable;
import java.util.concurrent.atomic.AtomicLong;
/**
- * Create handle with RowData for datasource implemention of bulk insert.
+ * Create handle with RowData for datasource implementation of bulk insert.
*/
public class HoodieRowDataCreateHandle implements Serializable {
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java
index 553b084e29c..56283dcb4d5 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkFileReader.java
@@ -21,7 +21,7 @@ package org.apache.hudi.io.storage;
import org.apache.spark.sql.catalyst.InternalRow;
/**
- * Marker interface for every {@link HoodieFileReader} reading in Catalyst (Spark native tyeps, ie
+ * Marker interface for every {@link HoodieFileReader} reading in Catalyst (Spark native types, i.e.,
* producing {@link InternalRow}s)
*/
public interface HoodieSparkFileReader extends HoodieFileReader<InternalRow> {}
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala
index f05da8a0ce8..290b118bd89 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/DataFrameUtil.scala
@@ -27,7 +27,7 @@ object DataFrameUtil {
/**
* Creates a DataFrame out of RDD[InternalRow] that you can get
- * using `df.queryExection.toRdd`
+ * using `df.queryExecution.toRdd`
*/
def createFromInternalRows(sparkSession: SparkSession, schema:
StructType, rdd: RDD[InternalRow]): DataFrame = {
diff --git a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala
index 7c39ce2546f..f00bb90a441 100644
--- a/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala
+++ b/hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/hudi/execution/RangeSample.scala
@@ -340,7 +340,7 @@ object RangeSampleSort {
val longBound = bound.asInstanceOf[Array[Long]]
for (i <- 0 to bound.length - 1) {
for (j <- 0 to fillFactor - 1) {
- // sample factor shoud not be too large, so it's ok to use 1 / fillfactor as slice
+ // sample factor should not be too large, so it's ok to use 1 / fillfactor as slice
newBound(j + i*(fillFactor)) = longBound(i) + (j + 1) * (1 / fillFactor.toDouble)
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
index 3a5e940d460..96933c261a8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestClientRollback.java
@@ -204,7 +204,7 @@ public class TestClientRollback extends HoodieClientTestBase {
//restore again
newClient.restoreToSavepoint(savepoint.getTimestamp());
- //verify that we resuse the existing restore commit
+ //verify that we reuse the existing restore commit
metaClient = HoodieTableMetaClient.reload(metaClient);
table = HoodieSparkTable.create(getConfig(), context, metaClient);
List<HoodieInstant> restoreInstants = table.getActiveTimeline().getRestoreTimeline().getInstants();
@@ -469,7 +469,7 @@ public class TestClientRollback extends HoodieClientTestBase {
}
/**
- * Test Cases for effects of rollbacking completed/inflight commits.
+ * Test Cases for effects of rolling back completed/inflight commits.
*/
@ParameterizedTest
@MethodSource("testFailedRollbackCommitParams")
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
index 51897255968..27774e59642 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
@@ -208,7 +208,7 @@ public class TestHoodieClientMultiWriter extends HoodieClientTestBase {
final JavaRDD<WriteStatus> writeStatusList2 = startCommitForUpdate(writeConfig, client2, nextCommitTime2, 100);
// start to write commit 003
- // this commit 003 will failed quickly because early conflict detection before create marker.
+ // this commit 003 will fail quickly because early conflict detection before create marker.
final String nextCommitTime3 = "003";
assertThrows(SparkException.class, () -> {
final JavaRDD<WriteStatus> writeStatusList3 = startCommitForUpdate(writeConfig, client3, nextCommitTime3, 100);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index 1fb623dbe84..94c3495b0ff 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -69,9 +69,9 @@ import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
-import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.ExternalSpillableMap;
import org.apache.hudi.common.util.hash.ColumnIndexID;
import org.apache.hudi.common.util.hash.PartitionIndexID;
@@ -625,7 +625,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
doCleanAndValidate(testTable, "0000003", Arrays.asList("0000001"));
HoodieTableMetadata tableMetadata = metadata(writeConfig, context);
- // since clean was the last commit, table servives should not get triggered in metadata table.
+ // since clean was the last commit, table services should not get triggered in metadata table.
assertFalse(tableMetadata.getLatestCompactionTime().isPresent());
doWriteOperation(testTable, "0000004", UPSERT);
@@ -1460,7 +1460,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
/**
* Tests the metadata payload spurious deletes.
- * Lets say a commit was applied to metadata table, and later was explicitly got rolledback. Due to spark task failures, there could be more files in rollback
+ * Lets say a commit was applied to metadata table, and later was explicitly got rolled back. Due to spark task failures, there could be more files in rollback
* metadata when compared to the original commit metadata. When payload consistency check is enabled, it will throw exception. If not, it will succeed.
*
* @throws Exception
@@ -1558,7 +1558,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
Map<String, Map<String, List<String>>> commitToPartitionsToFiles = new HashMap<>();
- // populate commit -> partition -> file info to assist in validation and prefi
+ // populate commit -> partition -> file info to assist in validation and prefix search
metaClient.getActiveTimeline().getInstants().forEach(entry -> {
try {
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
@@ -2095,7 +2095,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
/**
- * Instants on Metadata Table should be archived as per config but we always keep atlest the number of instants
+ * Instants on Metadata Table should be archived as per config but we always keep at least the number of instants
* as on the dataset.
* <p>
* Metadata Table should be automatically compacted as per config.
@@ -2159,7 +2159,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.startCommitWithTime(newCommitTime);
client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
if (i == 0) {
- // Mark this commit inflight so compactions dont take place
+ // Mark this commit inflight so compactions don't take place
FileCreateUtils.deleteCommit(basePath, newCommitTime);
FileCreateUtils.createInflightCommit(basePath, newCommitTime);
inflightCommitTime = newCommitTime;
@@ -2246,7 +2246,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
/**
- * When table needs to be upgraded and when multi writer is enabled, hudi rollsback partial commits. Upgrade itself is happening
+ * When table needs to be upgraded and when multi writer is enabled, hudi rolls back partial commits. Upgrade itself is happening
* within a lock and hence rollback should not lock again.
*
* @throws IOException
@@ -2320,7 +2320,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
/**
- * Tests rollback of a commit which has new partitions which is not present in hudi table prior to the commit being rolledback.
+ * Tests rollback of a commit which has new partitions which is not present in hudi table prior to the commit being rolled back.
*
* @throws Exception
*/
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index a5ccffa7e30..71d45785d4e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -447,7 +447,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* Test Deduplication Logic for write function.
*
- * @param writeFn One of HoddieWriteClient non-prepped write APIs
+ * @param writeFn One of HoodieWriteClient non-prepped write APIs
* @throws Exception in case of failure
*/
private void testDeduplication(
@@ -1018,7 +1018,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
/**
* This test ensures in a global bloom when update partition path is set to true in config, if an incoming record has mismatched partition
- * compared to whats in storage, then appropriate actions are taken. i.e. old record is deleted in old partition and new one is inserted
+ * compared to what is in storage, then appropriate actions are taken. i.e. old record is deleted in old partition and new one is inserted
* in the new partition.
* test structure:
* 1. insert 1 batch
@@ -1555,7 +1555,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
replaceCommitMetadata.getPartitionToWriteStats()
.forEach((k,v) -> v.forEach(entry -> filesFromReplaceCommit.add(entry.getPath())));
- // find all parquet files created as part of clustering. Verify it matces w/ whats found in replace commit metadata.
+ // find all parquet files created as part of clustering. Verify it matches w/ what is found in replace commit metadata.
FileStatus[] fileStatuses = fs.listStatus(new Path(basePath + "/" + partitionPath));
List<String> clusteredFiles = Arrays.stream(fileStatuses).filter(entry -> entry.getPath().getName().contains(replaceCommitInstant.getTimestamp()))
.map(fileStatus -> partitionPath + "/" + fileStatus.getPath().getName()).collect(Collectors.toList());
@@ -2474,7 +2474,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
HoodieTestUtils.init(hadoopConf, basePath);
SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, populateMetaFields));
- // perform 1 successfull commit
+ // perform 1 successful commit
writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
0, true);
@@ -2612,7 +2612,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
ExecutorService service = Executors.newFixedThreadPool(2);
HoodieTestUtils.init(hadoopConf, basePath);
SparkRDDWriteClient client = new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true));
- // perform 1 successfull write
+ // perform 1 successful write
writeBatch(client, "100", "100", Option.of(Arrays.asList("100")), "100",
100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 100,
0, true);
@@ -2629,7 +2629,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
client.close();
// refresh data generator to delete records generated from failed commits
dataGen = new HoodieTestDataGenerator();
- // Create a succesful commit
+ // Create a successful commit
Future<JavaRDD<WriteStatus>> commit3 = service.submit(() -> writeBatch(new SparkRDDWriteClient(context, getParallelWritingWriteConfig(cleaningPolicy, true)),
"400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
index 8531030a5cc..a0bee7bb37a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieMetadataBootstrap.java
@@ -106,7 +106,7 @@ public class TestHoodieMetadataBootstrap extends TestHoodieMetadataBase {
// validate
validateMetadata(testTable);
- // after bootstrap do two writes and validate its still functional.
+ // after bootstrap do two writes and validate it's still functional.
doWriteInsertAndUpsert(testTable);
validateMetadata(testTable);
}
@@ -178,7 +178,7 @@ public class TestHoodieMetadataBootstrap extends TestHoodieMetadataBase {
// once the commit is complete, metadata should get fully synced.
// in prod code path, SparkHoodieBackedTableMetadataWriter.create() will be called for every commit,
// which may not be the case here if we directly call HoodieBackedTableMetadataWriter.update()
- // hence lets first move the commit to complete and invoke sync directly
+ // hence let's first move the commit to complete and invoke sync directly
((HoodieMetadataTestTable) testTable).moveInflightCommitToComplete("00000007", inflightCommitMeta, true);
syncTableMetadata(writeConfig);
validateMetadata(testTable);
@@ -252,7 +252,7 @@ public class TestHoodieMetadataBootstrap extends TestHoodieMetadataBase {
initWriteConfigAndMetatableWriter(writeConfig, true);
syncTableMetadata(writeConfig);
validateMetadata(testTable);
- // after bootstrap do two writes and validate its still functional.
+ // after bootstrap do two writes and validate it's still functional.
doWriteInsertAndUpsert(testTable);
validateMetadata(testTable);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java
index 830577463b2..544d3e2930e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestSimpleExecutionInSpark.java
@@ -18,7 +18,6 @@
package org.apache.hudi.execution;
-import org.apache.avro.generic.IndexedRecord;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
@@ -28,6 +27,8 @@ import org.apache.hudi.common.util.queue.HoodieConsumer;
import org.apache.hudi.common.util.queue.SimpleExecutor;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.testutils.HoodieClientTestHarness;
+
+import org.apache.avro.generic.IndexedRecord;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -166,7 +167,7 @@ public class TestSimpleExecutionInSpark extends HoodieClientTestHarness {
}
/**
- * Test to ensure exception happend in iterator then we need to stop the simple ingestion.
+ * Test to ensure exception happen in iterator then we need to stop the simple ingestion.
*/
@SuppressWarnings("unchecked")
@Test
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
index 5fa59509091..f1dc74215a8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
@@ -21,6 +21,7 @@ package org.apache.hudi.index.hbase;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
import org.apache.hudi.client.common.HoodieSparkEngineContext;
+import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
import org.apache.hudi.common.model.HoodieAvroRecord;
import org.apache.hudi.common.model.HoodieKey;
@@ -32,12 +33,11 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieArchivalConfig;
+import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieHBaseIndexConfig;
import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.table.HoodieSparkTable;
@@ -78,6 +78,9 @@ import java.util.stream.Collectors;
import scala.Tuple2;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
+import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -87,9 +90,6 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_CLIENT_PORT;
-import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
-import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_QUORUM;
/**
* Note :: HBaseTestingUtility is really flaky with issues where the HbaseMiniCluster fails to shutdown across tests,
@@ -286,15 +286,15 @@ public class TestSparkHoodieHBaseIndex extends SparkClientFunctionalTestHarness
public void testTagLocationAndPartitionPathUpdateWithExplicitRollback() throws Exception {
final int numRecords = 10;
final String oldPartitionPath = "1970/01/01";
- final String emptyHoodieRecordPayloadClasssName = EmptyHoodieRecordPayload.class.getName();
- HoodieWriteConfig config = getConfigBuilder(100, true, true).withRollbackUsingMarkers(false).build();
+ final String emptyHoodieRecordPayloadClassName = EmptyHoodieRecordPayload.class.getName();
+ HoodieWriteConfig config = getConfigBuilder(100, true, true).withRollbackUsingMarkers(false).build();
SparkHoodieHBaseIndex index = new SparkHoodieHBaseIndex(config);
try (SparkRDDWriteClient writeClient = getHoodieWriteClient(config);) {
final String firstCommitTime = writeClient.startCommit();
List<HoodieRecord> newRecords = dataGen.generateInserts(firstCommitTime, numRecords);
List<HoodieRecord> oldRecords = new LinkedList();
- for (HoodieRecord newRecord: newRecords) {
+ for (HoodieRecord newRecord : newRecords) {
HoodieKey key = new HoodieKey(newRecord.getRecordKey(), oldPartitionPath);
HoodieRecord hoodieRecord = new HoodieAvroRecord(key, (HoodieRecordPayload) newRecord.getData());
oldRecords.add(hoodieRecord);
@@ -326,8 +326,8 @@ public class TestSparkHoodieHBaseIndex extends SparkClientFunctionalTestHarness
assertEquals(numRecords, afterFirstTaggedRecords.stream().filter(HoodieRecord::isCurrentLocationKnown).count());
// Verify the second commit
assertEquals(numRecords, beforeSecondTaggedRecords.stream()
- .filter(record -> record.getKey().getPartitionPath().equals(oldPartitionPath)
- && record.getData().getClass().getName().equals(emptyHoodieRecordPayloadClasssName)).count());
+ .filter(record -> record.getKey().getPartitionPath().equals(oldPartitionPath)
+ && record.getData().getClass().getName().equals(emptyHoodieRecordPayloadClassName)).count());
assertEquals(numRecords * 2, beforeSecondTaggedRecords.stream().count());
assertEquals(numRecords, afterSecondTaggedRecords.stream().count());
assertEquals(numRecords, afterSecondTaggedRecords.stream().filter(record -> !record.getKey().getPartitionPath().equals(oldPartitionPath)).count());
@@ -335,7 +335,7 @@ public class TestSparkHoodieHBaseIndex extends SparkClientFunctionalTestHarness
// If an exception occurs after hbase writes the index and the index does not roll back,
// the currentLocation information will not be returned.
assertEquals(numRecords, afterRollback.stream().filter(record -> record.getKey().getPartitionPath().equals(oldPartitionPath)
- && record.getData().getClass().getName().equals(emptyHoodieRecordPayloadClasssName)).count());
+ && record.getData().getClass().getName().equals(emptyHoodieRecordPayloadClassName)).count());
assertEquals(numRecords * 2, beforeSecondTaggedRecords.stream().count());
assertEquals(numRecords, afterRollback.stream().filter(HoodieRecord::isCurrentLocationKnown)
.filter(record -> record.getCurrentLocation().getInstantTime().equals(firstCommitTime)).count());
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
index 6aac7012a04..70115c3bfa3 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
@@ -140,7 +140,7 @@ public class TestInlineCompaction extends CompactionTestBase {
//should be here
}
- // step 3: compelete another 4 delta commit should be 2 compaction request after this
+ // step 3: complete another 4 delta commit should be 2 compaction request after this
instants = IntStream.range(0, 4).mapToObj(i -> HoodieActiveTimeline.createNewInstantTime()).collect(Collectors.toList());
records = dataGen.generateInsertsForPartition(instants.get(0), 100, "2022/03/15");
for (String instant : instants) {
@@ -295,7 +295,7 @@ public class TestInlineCompaction extends CompactionTestBase {
}
// When: commit happens after 1000s. assumption is that, there won't be any new compaction getting scheduled within 100s, but the previous failed one will be
- // rolledback and retried to move it to completion.
+ // rolled back and retried to move it to completion.
HoodieWriteConfig inlineCfg = getConfigForInlineCompaction(5, 1000, CompactionTriggerStrategy.TIME_ELAPSED);
String instantTime2;
try (SparkRDDWriteClient<?> writeClient = getHoodieWriteClient(inlineCfg)) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
index 319d6ea031e..2e08dfe398d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/strategy/TestHoodieCompactionStrategy.java
@@ -262,7 +262,7 @@ public class TestHoodieCompactionStrategy {
"LogFileLengthBasedCompactionStrategy should have resulted in fewer compactions");
assertEquals(2, returned.size(), "LogFileLengthBasedCompactionStrategy should have resulted in 2 compaction");
- // Delte log File length
+ // Delta log File length
Integer allFileLength = returned.stream().map(s -> s.getDeltaFilePaths().size())
.reduce(Integer::sum).orElse(0);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java
index d2927f2be5a..984dce3b09c 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableRollback.java
@@ -21,6 +21,7 @@ package org.apache.hudi.table.functional;
import org.apache.hudi.client.SparkRDDWriteClient;
import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieBaseFile;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -46,7 +47,6 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.index.HoodieIndex;
@@ -556,7 +556,7 @@ public class TestHoodieSparkMergeOnReadTableRollback extends SparkClientFunction
upsertRecords(client, "010", records, dataGen);
- // trigger clean. creating a new client with aggresive cleaner configs so that clean will kick in immediately.
+ // trigger clean. creating a new client with aggressive cleaner configs so that clean will kick in immediately.
cfgBuilder = getConfigBuilder(false)
.withCleanConfig(HoodieCleanConfig.newBuilder().retainCommits(1).build())
// Timeline-server-based markers are not used for multi-rollback tests
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
index 68f7a9f16c1..cbf30fb2a7f 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/ColumnStatsIndexSupport.scala
@@ -357,7 +357,7 @@ object ColumnStatsIndexSupport {
"BytesWrapper", "StringWrapper", "DateWrapper", "DecimalWrapper", "TimeMicrosWrapper", "TimestampMicrosWrapper")
/**
- * Target Column Stats Index columns which internally are mapped onto fields of the correspoding
+ * Target Column Stats Index columns which internally are mapped onto fields of the corresponding
* Column Stats record payload ([[HoodieMetadataColumnStats]]) persisted w/in Metadata Table
*/
private val targetColumnStatsIndexColumns = Seq(
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
index e0f46a5e77e..2c08afaa691 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
@@ -19,9 +19,8 @@
package org.apache.hudi
import org.apache.hadoop.fs.Path
-import org.apache.hudi.HoodieBaseRelation.{BaseFileReader, projectReader}
+import org.apache.hudi.HoodieBaseRelation.{BaseFileReader, convertToAvroSchema, projectReader}
import org.apache.hudi.HoodieBootstrapRelation.validate
-import org.apache.hudi.HoodieBaseRelation.convertToAvroSchema
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.spark.rdd.RDD
@@ -29,7 +28,7 @@ import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.execution.datasources.PartitionedFile
-import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.{isMetaField, removeMetaFields}
+import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.isMetaField
import org.apache.spark.sql.sources.Filter
import org.apache.spark.sql.types.StructType
@@ -37,21 +36,21 @@ case class HoodieBootstrapSplit(dataFile: PartitionedFile, skeletonFile: Option[
/**
* This is Spark relation that can be used for querying metadata/fully bootstrapped query hoodie tables, as well as
- * non-bootstrapped tables. It implements PrunedFilteredScan interface in order to support column pruning and filter
- * push-down. For metadata bootstrapped files, if we query columns from both metadata and actual data then it will
- * perform a merge of both to return the result.
- *
- * Caveat: Filter push-down does not work when querying both metadata and actual data columns over metadata
- * bootstrapped files, because then the metadata file and data file can return different number of rows causing errors
- * merging.
- *
- * @param sqlContext Spark SQL Context
- * @param userSchema User specified schema in the datasource query
- * @param globPaths The global paths to query. If it not none, read from the globPaths,
- * else read data from tablePath using HoodiFileIndex.
- * @param metaClient Hoodie table meta client
- * @param optParams DataSource options passed by the user
- */
+ * non-bootstrapped tables. It implements PrunedFilteredScan interface in order to support column pruning and filter
+ * push-down. For metadata bootstrapped files, if we query columns from both metadata and actual data then it will
+ * perform a merge of both to return the result.
+ *
+ * Caveat: Filter push-down does not work when querying both metadata and actual data columns over metadata
+ * bootstrapped files, because then the metadata file and data file can return different number of rows causing errors
+ * merging.
+ *
+ * @param sqlContext Spark SQL Context
+ * @param userSchema User specified schema in the datasource query
+ * @param globPaths The global paths to query. If it not none, read from the globPaths,
+ * else read data from tablePath using HoodieFileIndex.
+ * @param metaClient Hoodie table meta client
+ * @param optParams DataSource options passed by the user
+ */
case class HoodieBootstrapRelation(override val sqlContext: SQLContext,
private val userSchema: Option[StructType],
private val globPaths: Seq[Path],
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index 8724750f798..1eb5bc42924 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -473,7 +473,7 @@ object HoodieSparkSqlWriter {
// only incoming dataset's projection has to match the table's schema, and not the whole one
if (mergeIntoWrites) {
- // if its merge into writes, do not check for projection nor schema compatability. Writers down the line will
+ // if its merge into writes, do not check for projection nor schema compatibility. Writers down the line will
// take care of it.
canonicalizedSourceSchema
} else {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
index 502d25f8287..ec457975f1c 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieStreamingSink.scala
@@ -23,15 +23,15 @@ import org.apache.hudi.HoodieSinkCheckpoint.SINK_CHECKPOINT_KEY
import org.apache.hudi.async.{AsyncClusteringService, AsyncCompactService, SparkStreamingAsyncClusteringService, SparkStreamingAsyncCompactService}
import org.apache.hudi.client.SparkRDDWriteClient
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieRecordPayload, WriteConcurrencyMode}
-import org.apache.hudi.config.HoodieWriteConfig.WRITE_CONCURRENCY_MODE
+import org.apache.hudi.common.model.{HoodieCommitMetadata, WriteConcurrencyMode}
import org.apache.hudi.common.table.marker.MarkerType
import org.apache.hudi.common.table.timeline.HoodieInstant.State
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
-import org.apache.hudi.common.util.ValidationUtils.{checkArgument, checkState}
+import org.apache.hudi.common.util.ValidationUtils.checkArgument
import org.apache.hudi.common.util.{ClusteringUtils, CommitUtils, CompactionUtils, JsonUtils, StringUtils}
-import org.apache.hudi.config.{HoodieLockConfig, HoodieWriteConfig}
+import org.apache.hudi.config.HoodieWriteConfig
+import org.apache.hudi.config.HoodieWriteConfig.WRITE_CONCURRENCY_MODE
import org.apache.hudi.exception.{HoodieCorruptedDataException, HoodieException, TableNotFoundException}
import org.apache.log4j.LogManager
import org.apache.spark.api.java.JavaSparkContext
@@ -330,7 +330,7 @@ class HoodieStreamingSink(sqlContext: SQLContext,
if (!DELETE_OPERATION_OPT_VAL.equals(operationType)) {
getStreamIdentifier(options) match {
case Some(identifier) =>
- // get the latest checkpoint from the commit metadata to check if the microbatch has already been prcessed or not
+ // get the latest checkpoint from the commit metadata to check if the microbatch has already been processed or not
val commitMetadata = CommitUtils.getLatestCommitMetadataWithValidCheckpointInfo(
metaClient.get.getActiveTimeline.getCommitsTimeline, SINK_CHECKPOINT_KEY)
if (commitMetadata.isPresent) {
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala
index 3e541bb09a0..ad1e87f8ce0 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/execution/datasources/HoodieInMemoryFileIndex.scala
@@ -24,8 +24,8 @@ import org.apache.hudi.SparkAdapterSupport
import org.apache.spark.HoodieHadoopFSUtils
import org.apache.spark.metrics.source.HiveCatalogMetrics
import org.apache.spark.sql.SparkSession
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, Expression}
import org.apache.spark.sql.catalyst.{InternalRow, expressions}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, BoundReference, Expression, InterpretedPredicate}
import org.apache.spark.sql.execution.datasources._
import org.apache.spark.sql.types.StructType
@@ -41,7 +41,7 @@ class HoodieInMemoryFileIndex(sparkSession: SparkSession,
with SparkAdapterSupport {
/**
- * Returns all valid files grouped into partitions when the data is partitioned. If the data is unpartitioned,
+ * Returns all valid files grouped into partitions when the data is partitioned. If the data is non-partitioned,
* this will return a single partition with no partition values
*
* NOTE: This method replicates the one it overrides, however it uses custom method
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
index 826c0090b21..09eb9f2944f 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/spark/sql/catalyst/catalog/HoodieCatalogTable.scala
@@ -94,7 +94,7 @@ class HoodieCatalogTable(val spark: SparkSession, var table: CatalogTable) exten
lazy val tableName: String = tableConfig.getTableName
/**
- * the name of dabase
+ * the name of database
*/
lazy val databaseName: String = tableConfig.getDatabaseName
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala
index 418b0c17adb..e88e1c12798 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/MergeIntoHoodieTableCommand.scala
@@ -37,12 +37,12 @@ import org.apache.spark.sql.catalyst.expressions.BindReferences.bindReference
import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, BoundReference, EqualTo, Expression, Literal, NamedExpression, PredicateHelper}
import org.apache.spark.sql.catalyst.plans.logical._
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils._
-import org.apache.spark.sql.hudi.analysis.HoodieAnalysis.failAnalysis
+import org.apache.spark.sql.hudi.ProvidesHoodieConfig
import org.apache.spark.sql.hudi.ProvidesHoodieConfig.combineOptions
+import org.apache.spark.sql.hudi.analysis.HoodieAnalysis.failAnalysis
import org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand.{CoercedAttributeReference, encodeAsBase64String, stripCasting, toStructType}
import org.apache.spark.sql.hudi.command.payload.ExpressionPayload
import org.apache.spark.sql.hudi.command.payload.ExpressionPayload._
-import org.apache.spark.sql.hudi.ProvidesHoodieConfig
import org.apache.spark.sql.types.{BooleanType, StructField, StructType}
import java.util.Base64
@@ -272,29 +272,29 @@ case class MergeIntoHoodieTableCommand(mergeInto: MergeIntoTable) extends Hoodie
* will be applied:
*
* <ul>
- * <li>Expression for the "primary-key" column is extracted from the merge-on condition of the
- * MIT statement: Hudi's implementation of the statement restricts kind of merge-on condition
- * permitted to only such referencing primary-key column(s) of the target table; as such we're
- * leveraging matching side of such conditional expression (containing [[sourceTable]] attrobute)
- * interpreting it as a primary-key column in the [[sourceTable]]</li>
+ * <li>Expression for the "primary-key" column is extracted from the merge-on condition of the
+ * MIT statement: Hudi's implementation of the statement restricts kind of merge-on condition
+ * permitted to only such referencing primary-key column(s) of the target table; as such we're
+ * leveraging matching side of such conditional expression (containing [[sourceTable]] attribute)
+ * interpreting it as a primary-key column in the [[sourceTable]]</li>
*
- * <li>Expression for the "pre-combine" column (optional) is extracted from the matching update
- * clause ({@code WHEN MATCHED ... THEN UPDATE ...}) as right-hand side of the expression referencing
- * pre-combine attribute of the target column</li>
+ * <li>Expression for the "pre-combine" column (optional) is extracted from the matching update
+ * clause ({@code WHEN MATCHED ... THEN UPDATE ...}) as right-hand side of the expression referencing
+ * pre-combine attribute of the target column</li>
* <ul>
*
* For example, w/ the following statement (primary-key column is [[id]], while pre-combine column is [[ts]])
* <pre>
- * MERGE INTO target
- * USING (SELECT 1 AS sid, 'A1' AS sname, 1000 AS sts) source
- * ON target.id = source.sid
- * WHEN MATCHED THEN UPDATE SET id = source.sid, name = source.sname, ts = source.sts
+ * MERGE INTO target
+ * USING (SELECT 1 AS sid, 'A1' AS sname, 1000 AS sts) source
+ * ON target.id = source.sid
+ * WHEN MATCHED THEN UPDATE SET id = source.sid, name = source.sname, ts = source.sts
* </pre>
*
* We will append following columns to the source dataset:
* <ul>
- * <li>{@code id = source.sid}</li>
- * <li>{@code ts = source.sts}</li>
+ * <li>{@code id = source.sid}</li>
+ * <li>{@code ts = source.sts}</li>
* </ul>
*/
def sourceDataset: DataFrame = {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala
index 11f45c11386..ad1cb3b3535 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/payload/ExpressionPayload.scala
@@ -26,8 +26,7 @@ import org.apache.hudi.SparkAdapterSupport.sparkAdapter
import org.apache.hudi.avro.AvroSchemaUtils.{isNullable, resolveNullableSchema}
import org.apache.hudi.avro.HoodieAvroUtils
import org.apache.hudi.avro.HoodieAvroUtils.bytesToAvro
-import org.apache.hudi.common.model.BaseAvroPayload
-import org.apache.hudi.common.model.{DefaultHoodieRecordPayload, HoodiePayloadProps, HoodieRecord}
+import org.apache.hudi.common.model.{BaseAvroPayload, DefaultHoodieRecordPayload, HoodiePayloadProps, HoodieRecord}
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.hudi.common.util.{BinaryUtil, ValidationUtils, Option => HOption}
import org.apache.hudi.config.HoodieWriteConfig
@@ -299,12 +298,12 @@ object ExpressionPayload {
val PAYLOAD_DELETE_CONDITION = "hoodie.payload.delete.condition"
/**
- * Property for pass the merge-into update clauses's condition and assignments.
+ * Property for pass the merge-into update clauses' condition and assignments.
*/
val PAYLOAD_UPDATE_CONDITION_AND_ASSIGNMENTS = "hoodie.payload.update.condition.assignments"
/**
- * Property for pass the merge-into insert clauses's condition and assignments.
+ * Property for pass the merge-into insert clauses' condition and assignments.
*/
val PAYLOAD_INSERT_CONDITION_AND_ASSIGNMENTS = "hoodie.payload.insert.condition.assignments"
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index dc9bff48f38..2101bc2a979 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -18,10 +18,6 @@
package org.apache.hudi
import org.apache.avro.Schema
-
-import java.io.IOException
-import java.time.Instant
-import java.util.{Collections, Date, UUID}
import org.apache.commons.io.FileUtils
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.HoodieSparkUtils.gteqSpark3_0
@@ -36,12 +32,12 @@ import org.apache.hudi.functional.TestBootstrap
import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator, SimpleKeyGenerator}
import org.apache.hudi.testutils.DataSourceTestUtils
import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest
+import org.apache.spark.SparkContext
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql._
import org.apache.spark.sql.functions.{expr, lit}
import org.apache.spark.sql.hudi.HoodieSparkSessionExtension
import org.apache.spark.sql.hudi.command.SqlKeyGenerator
-import org.apache.spark.{SparkConf, SparkContext}
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue, fail}
import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
import org.junit.jupiter.params.ParameterizedTest
@@ -1045,7 +1041,7 @@ class TestHoodieSparkSqlWriter {
}
@Test
- def testNonpartitonedWithResuseTableConfig(): Unit = {
+ def testNonpartitonedWithReuseTableConfig(): Unit = {
val _spark = spark
import _spark.implicits._
val df = Seq((1, "a1", 10, 1000, "2021-10-16")).toDF("id", "name", "value", "ts", "dt")
@@ -1054,7 +1050,7 @@ class TestHoodieSparkSqlWriter {
DataSourceWriteOptions.PRECOMBINE_FIELD.key -> "ts"
)
- // case 1: When commit C1 specificies a key generator and commit C2 does not specify key generator
+ // case 1: When commit C1 specifies a key generator and commit C2 does not specify key generator
val (tableName1, tablePath1) = ("hoodie_test_params_1", s"$tempBasePath" + "_1")
// the first write need to specify KEYGENERATOR_CLASS_NAME params
@@ -1084,7 +1080,7 @@ class TestHoodieSparkSqlWriter {
DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "dt"
)
- // case 1: When commit C1 does not specify key generator and commit C2 specificies a key generator
+ // case 1: When commit C1 does not specify key generator and commit C2 specifies a key generator
val (tableName1, tablePath1) = ("hoodie_test_params_1", s"$tempBasePath" + "_1")
// the first write need to specify KEYGENERATOR_CLASS_NAME params
@@ -1118,7 +1114,7 @@ class TestHoodieSparkSqlWriter {
DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "dt"
)
- // case 1: When commit C1 specificies a key generator and commkt C2 does not specify key generator
+ // case 1: When commit C1 specifies a key generator and commkt C2 does not specify key generator
val (tableName1, tablePath1) = ("hoodie_test_params_1", s"$tempBasePath" + "_1")
// the first write need to specify KEYGENERATOR_CLASS_NAME params
@@ -1151,7 +1147,7 @@ class TestHoodieSparkSqlWriter {
DataSourceWriteOptions.PARTITIONPATH_FIELD.key -> "dt"
)
- // case 1: When commit C1 specificies a key generator and commkt C2 does not specify key generator
+ // case 1: When commit C1 specifies a key generator and commkt C2 does not specify key generator
val (tableName1, tablePath1) = ("hoodie_test_params_1", s"$tempBasePath" + "_1")
// the first write need to specify KEYGENERATOR_CLASS_NAME params
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
index 107514c1de8..0018d90a3b3 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
@@ -68,11 +68,11 @@ class TestMORDataSourceStorage extends SparkClientFunctionalTestHarness {
"false,",
"false,fare.currency"
))
- def testMergeOnReadStorage(isMetadataEnabled: Boolean, preComineField: String) {
+ def testMergeOnReadStorage(isMetadataEnabled: Boolean, preCombineField: String) {
var options: Map[String, String] = commonOpts +
(HoodieMetadataConfig.ENABLE.key -> String.valueOf(isMetadataEnabled))
- if (!StringUtils.isNullOrEmpty(preComineField)) {
- options += (DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> preComineField)
+ if (!StringUtils.isNullOrEmpty(preCombineField)) {
+ options += (DataSourceWriteOptions.PRECOMBINE_FIELD.key() -> preCombineField)
}
val dataGen = new HoodieTestDataGenerator(0xDEEF)
val fs = FSUtils.getFs(basePath, spark.sparkContext.hadoopConfiguration)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
index 14b4f50700a..0d5ee362a8b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
@@ -41,7 +41,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
* Step3: Delete 20 With Clustering
* Step4: Insert Overwrite 50
* Step5: Upsert 30 With Clean
- * Step6: Bluk_Insert 20
+ * Step6: Bulk_Insert 20
*/
@ParameterizedTest
@EnumSource(classOf[HoodieCDCSupplementalLoggingMode])
@@ -218,7 +218,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
* Step1: Insert 100
* Step2: Upsert 50
* Step3: Delete 20 With Compaction
- * Step4: Bluk_Insert 100
+ * Step4: Bulk_Insert 100
* Step5: Upsert 60 With Clustering
* Step6: Insert Overwrite 70
* Step7: Upsert 30 With CLean
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala
index 4ef3bea6344..3d172fecdf6 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/avro/TestSchemaConverters.scala
@@ -32,7 +32,7 @@ class TestSchemaConverters {
val convertedAvroSchema = SchemaConverters.toAvroType(convertedStructType)
// NOTE: Here we're validating that converting Avro -> Catalyst and Catalyst -> Avro are inverse
- // transformations, but since it's not an easy endeavor to match Avro scehams, we match
+ // transformations, but since it's not an easy endeavor to match Avro schemas, we match
// derived Catalyst schemas instead
assertEquals(convertedStructType, SchemaConverters.toSqlType(convertedAvroSchema).dataType)
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
index cd46e9f3789..de77b07fd83 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestSpark3DDL.scala
@@ -23,10 +23,9 @@ import org.apache.hudi.QuickstartUtils.{DataGenerator, convertToStringList, getQ
import org.apache.hudi.common.config.HoodieStorageConfig
import org.apache.hudi.common.model.HoodieRecord
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator
-import org.apache.hudi.common.testutils.RawTripTestPayload
+import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, RawTripTestPayload}
import org.apache.hudi.config.HoodieWriteConfig
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieSparkRecordMerger, HoodieSparkUtils}
+import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkRecordMerger, HoodieSparkUtils}
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.functions.{arrays_zip, col, expr, lit}
import org.apache.spark.sql.types.StringType
@@ -245,7 +244,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
val meta = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
assert(meta.comment.get.equals("it is a hudi table"))
assert(Seq("key1", "key2").filter(meta.properties.contains(_)).size == 2)
- // test unset propertes
+ // test unset properties
spark.sql(s"alter table $tableName unset tblproperties(comment, 'key1', 'key2')")
val unsetMeta = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName))
assert(Seq("key1", "key2").filter(unsetMeta.properties.contains(_)).size == 0)
@@ -524,7 +523,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
spark.sql(s"alter table ${tableName} add columns(name string comment 'add name back' after userx," +
s" userx.name string comment 'add userx.name back' first, userx.score int comment 'add userx.score back' after age)")
- // query new columns: name, userx.name, userx.score, those field should not be readed.
+ // query new columns: name, userx.name, userx.score, those field should not be read.
checkAnswer(spark.sql(s"select name, userx.name, userx.score from ${tableName}").collect())(Seq(null, null, null))
// insert again