You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yi...@apache.org on 2024/05/15 09:08:16 UTC
(hudi) 07/32: [HUDI-7651] Add util methods for creating meta client (#11081)
This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch branch-0.x
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 586898b92561a2ae1a576a425342684c14a6b3c2
Author: Y Ethan Guo <et...@gmail.com>
AuthorDate: Wed May 15 01:50:35 2024 -0700
[HUDI-7651] Add util methods for creating meta client (#11081)
---
.../hudi/cli/commands/TestMetadataCommand.java | 3 +-
.../hudi/testutils/HoodieMergeOnReadTestUtils.java | 4 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 33 +++++++-------
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 16 +++----
.../testutils/HoodieJavaClientTestHarness.java | 24 +++++-----
.../hudi/testutils/TestHoodieMetadataBase.java | 5 +++
.../hudi/client/TestCompactionAdminClient.java | 8 ++--
.../java/org/apache/hudi/client/TestMultiFS.java | 6 +--
.../functional/TestHoodieBackedMetadata.java | 41 +++++++----------
.../functional/TestHoodieBackedTableMetadata.java | 10 +----
.../TestHoodieClientOnCopyOnWriteStorage.java | 24 +++++-----
.../TestRemoteFileSystemViewWithMetadataTable.java | 5 +--
.../table/action/compact/CompactionTestBase.java | 18 ++++----
.../table/action/compact/TestInlineCompaction.java | 44 +++++++++---------
.../hudi/testutils/HoodieCleanerTestBase.java | 7 ++-
.../hudi/testutils/HoodieClientTestBase.java | 11 ++---
.../hudi/testutils/HoodieClientTestUtils.java | 28 +++++++++---
.../testutils/HoodieSparkClientTestHarness.java | 16 ++++++-
.../hudi/common/testutils/FileCreateUtils.java | 5 +--
.../common/testutils/HoodieCommonTestHarness.java | 2 +-
.../hudi/common/testutils/HoodieTestUtils.java | 41 +++++++++++++++++
.../hudi/common/util/TestCompactionUtils.java | 3 +-
.../sink/TestStreamWriteOperatorCoordinator.java | 10 ++---
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 4 +-
.../hudi/table/catalog/TestHoodieCatalog.java | 19 ++++----
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 23 ++++++----
.../apache/hudi/table/format/TestInputFormat.java | 10 +++--
.../test/java/org/apache/hudi/utils/TestData.java | 6 +--
.../org/apache/hudi/utils/TestStreamerUtil.java | 11 ++---
.../test/java/org/apache/hudi/utils/TestUtils.java | 33 +++++++-------
.../testsuite/job/TestHoodieTestSuiteJob.java | 13 +++---
.../hudi/command/procedures/BaseProcedure.scala | 6 +++
.../procedures/CommitsCompareProcedure.scala | 8 ++--
.../procedures/CreateMetadataTableProcedure.scala | 5 +--
.../procedures/CreateSavepointProcedure.scala | 4 +-
.../procedures/DeleteMetadataTableProcedure.scala | 4 +-
.../procedures/DeleteSavepointProcedure.scala | 4 +-
.../procedures/ExportInstantsProcedure.scala | 5 +--
.../procedures/InitMetadataTableProcedure.scala | 3 +-
.../RepairAddpartitionmetaProcedure.scala | 4 +-
.../RepairCorruptedCleanFilesProcedure.scala | 6 +--
.../RepairOverwriteHoodiePropsProcedure.scala | 9 ++--
.../procedures/RollbackToSavepointProcedure.scala | 4 +-
.../procedures/RunClusteringProcedure.scala | 4 +-
.../procedures/RunCompactionProcedure.scala | 4 +-
.../procedures/ShowArchivedCommitsProcedure.scala | 6 +--
.../procedures/ShowBootstrapMappingProcedure.scala | 3 +-
.../ShowBootstrapPartitionsProcedure.scala | 3 +-
.../procedures/ShowClusteringProcedure.scala | 7 ++-
.../ShowCommitExtraMetadataProcedure.scala | 4 +-
.../procedures/ShowCommitFilesProcedure.scala | 6 +--
.../procedures/ShowCommitPartitionsProcedure.scala | 6 +--
.../procedures/ShowCommitWriteStatsProcedure.scala | 6 +--
.../command/procedures/ShowCommitsProcedure.scala | 4 +-
.../procedures/ShowCompactionProcedure.scala | 4 +-
.../procedures/ShowFileSystemViewProcedure.scala | 2 +-
.../ShowHoodieLogFileMetadataProcedure.scala | 9 ++--
.../ShowHoodieLogFileRecordsProcedure.scala | 4 +-
.../ShowMetadataTableFilesProcedure.scala | 6 +--
.../ShowMetadataTableStatsProcedure.scala | 4 +-
.../procedures/ShowRollbacksProcedure.scala | 11 +++--
.../procedures/ShowSavepointsProcedure.scala | 4 +-
.../procedures/ShowTablePropertiesProcedure.scala | 3 +-
.../procedures/StatsFileSizeProcedure.scala | 4 +-
.../StatsWriteAmplificationProcedure.scala | 4 +-
.../procedures/ValidateHoodieSyncProcedure.scala | 6 +--
.../ValidateMetadataTableFilesProcedure.scala | 8 ++--
.../src/test/java/HoodieJavaStreamingApp.java | 3 +-
.../apache/hudi/functional/TestWriteClient.java | 2 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 19 +++-----
.../hudi/TestTableSchemaResolverWithSparkSQL.scala | 18 +++-----
.../hudi/functional/TestBasicSchemaEvolution.scala | 13 +-----
.../apache/hudi/functional/TestCOWDataSource.scala | 43 ++++++------------
.../hudi/functional/TestCOWDataSourceStorage.scala | 7 ++-
.../hudi/functional/TestHoodieActiveTimeline.scala | 9 ++--
.../TestIncrementalReadByStateTransitionTime.scala | 9 +---
.../TestIncrementalReadWithFullTableScan.scala | 9 ++--
.../hudi/functional/TestLayoutOptimization.scala | 14 +++---
.../apache/hudi/functional/TestMORDataSource.scala | 11 +----
.../hudi/functional/TestMORDataSourceStorage.scala | 8 ++--
.../functional/TestParquetColumnProjection.scala | 12 ++---
.../hudi/functional/TestSparkSqlCoreFlow.scala | 8 ++--
.../TestSparkSqlWithCustomKeyGenerator.scala | 8 ++--
.../hudi/functional/TestStructuredStreaming.scala | 31 +++++--------
.../hudi/functional/TestTimeTravelQuery.scala | 15 +++----
.../functional/cdc/TestCDCDataFrameSuite.scala | 38 +++++-----------
.../functional/cdc/TestCDCStreamingSuite.scala | 9 ++--
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 16 +++----
.../apache/spark/sql/hudi/common/TestSqlConf.scala | 6 +--
.../apache/spark/sql/hudi/ddl/TestAlterTable.scala | 22 +++------
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 6 +--
.../spark/sql/hudi/ddl/TestCreateTable.scala | 52 ++++++----------------
.../apache/spark/sql/hudi/ddl/TestSpark3DDL.scala | 12 ++---
.../spark/sql/hudi/dml/TestCDCForSparkSQL.scala | 18 +++-----
.../spark/sql/hudi/dml/TestInsertTable.scala | 23 +++-------
.../spark/sql/hudi/dml/TestMergeIntoTable2.scala | 8 ++--
.../spark/sql/hudi/dml/TestTimeTravelTable.scala | 34 ++++----------
.../spark/sql/hudi/dml/TestUpdateTable.scala | 9 ++--
.../hudi/procedure/TestBootstrapProcedure.scala | 5 +--
.../hudi/procedure/TestClusteringProcedure.scala | 11 ++---
.../hudi/procedure/TestCompactionProcedure.scala | 5 ++-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 37 ++++-----------
.../TestUpgradeOrDowngradeProcedure.scala | 13 ++----
.../org/apache/hudi/hive/TestHiveSyncTool.java | 5 ++-
104 files changed, 541 insertions(+), 694 deletions(-)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestMetadataCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestMetadataCommand.java
index 3214bb2cfcc..ca1d856f153 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestMetadataCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestMetadataCommand.java
@@ -46,6 +46,7 @@ import java.io.IOException;
import java.util.List;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
+import static org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -93,7 +94,7 @@ public class TestMetadataCommand extends CLIFunctionalTestHarness {
}
// verify that metadata partitions are filled in as part of table config.
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf()).setBasePath(tablePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(jsc(), tablePath);
assertFalse(metaClient.getTableConfig().getMetadataPartitions().isEmpty());
new TableCommand().connect(tablePath, null, false, 0, 0, 0);
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
index 7185115a4d5..51b27ba3661 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieMergeOnReadTestUtils.java
@@ -64,7 +64,7 @@ public class HoodieMergeOnReadTestUtils {
public static List<RecordReader> getRecordReadersUsingInputFormat(Configuration conf, List<String> inputPaths, String basePath, JobConf jobConf, boolean realtime, Schema rawSchema,
String rawHiveColumnTypes, boolean projectCols, List<String> projectedColumns, boolean populateMetaFields) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(conf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(conf, basePath);
FileInputFormat inputFormat = HoodieInputFormatUtils.getInputFormat(metaClient.getTableConfig().getBaseFileFormat(), realtime, jobConf);
Schema schema;
String hiveColumnTypes;
@@ -119,7 +119,7 @@ public class HoodieMergeOnReadTestUtils {
public static List<GenericRecord> getRecordsUsingInputFormat(Configuration conf, List<String> inputPaths, String basePath, JobConf jobConf, boolean realtime, Schema rawSchema,
String rawHiveColumnTypes, boolean projectCols, List<String> projectedColumns, boolean populateMetaFields) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(conf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(conf, basePath);
FileInputFormat inputFormat = HoodieInputFormatUtils.getInputFormat(metaClient.getTableConfig().getBaseFileFormat(), realtime, jobConf);
Schema schema;
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index a760723c4d2..0061017cb89 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -68,6 +68,7 @@ import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.JsonUtils;
import org.apache.hudi.common.util.Option;
@@ -354,7 +355,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// The earliest deltacommit in the metadata table should be "0000001",
// and the "00000000000000" init deltacommit should be archived.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
HoodieActiveTimeline metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals("0000001", metadataTimeline.getCommitsTimeline().firstInstant().get().getTimestamp());
@@ -366,7 +367,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
getHoodieWriteClient(writeConfig);
// Trigger a regular write operation. data set timeline archival should kick in.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
- archiveDataTable(writeConfig, HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build());
+ archiveDataTable(writeConfig, createMetaClient());
assertEquals("0000004",
metaClient.reloadActiveTimeline().getCommitsTimeline().firstInstant().get().getTimestamp());
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
@@ -405,13 +406,13 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// The earliest deltacommit in the metadata table should be "0000001",
// and the "00000000000000" init deltacommit should be archived.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
HoodieActiveTimeline metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals("0000001", metadataTimeline.getCommitsTimeline().firstInstant().get().getTimestamp());
getHoodieWriteClient(writeConfig);
// Trigger data table archive, should archive "0000001", "0000002"
- archiveDataTable(writeConfig, HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build());
+ archiveDataTable(writeConfig, createMetaClient());
// Trigger a regular write operation. metadata timeline archival should kick in and catch up with data table.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
@@ -537,7 +538,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
assertEquals(tableMetadata.getLatestCompactionTime().get(), "0000003001");
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
HoodieWriteConfig metadataTableWriteConfig = getMetadataWriteConfig(writeConfig);
metadataMetaClient.reloadActiveTimeline();
@@ -783,10 +784,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// 2nd commit
doWriteOperation(testTable, "0000001", INSERT);
- final HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
- .setConf(hadoopConf)
- .setBasePath(metadataTableBasePath)
- .build();
+ final HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
HoodieWriteConfig metadataTableWriteConfig = getMetadataWriteConfig(writeConfig);
metadataMetaClient.reloadActiveTimeline();
final HoodieTable table = HoodieJavaTable.create(metadataTableWriteConfig, context, metadataMetaClient);
@@ -1543,9 +1541,8 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
// ensure that 000003 is after rollback of the partially failed 2nd commit.
- HoodieTableMetaClient metadataMetaClient =
- HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/")
- .setConf(metaClient.getHadoopConf()).build();
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(
+ metaClient.getHadoopConf(), metaClient.getMetaPath() + "/metadata/");
HoodieInstant rollbackInstant =
metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
@@ -1716,7 +1713,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Ensure all commits were synced to the Metadata Table
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
assertEquals(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants().countInstants(), 5);
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "0000002")));
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "0000003")));
@@ -1765,7 +1762,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// Ensure all commits were synced to the Metadata Table
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
LOG.warn("total commits in metadata table " + metadataMetaClient.getActiveTimeline().getCommitsTimeline().countInstants());
// 6 commits and 2 cleaner commits.
@@ -2035,8 +2032,8 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
client.insert(records, newCommitTime);
}
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
- HoodieTableMetaClient datasetMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(config.getBasePath()).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
+ HoodieTableMetaClient datasetMetaClient = createMetaClient();
// There should not be any compaction yet and we have not performed more than maxDeltaCommitsBeforeCompaction
// deltacommits (1 will be due to bootstrap)
@@ -2640,7 +2637,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
// Execute compaction on metadata table.
try (JavaHoodieBackedTableMetadataWriter metadataWriter =
- (JavaHoodieBackedTableMetadataWriter) JavaHoodieBackedTableMetadataWriter.create(hadoopConf, client.getConfig(), context, Option.empty())) {
+ (JavaHoodieBackedTableMetadataWriter) JavaHoodieBackedTableMetadataWriter.create(hadoopConf, client.getConfig(), context, Option.empty())) {
Properties metadataProps = metadataWriter.getWriteConfig().getProps();
metadataProps.setProperty(INLINE_COMPACT_NUM_DELTA_COMMITS.key(), "3");
HoodieWriteConfig metadataWriteConfig = HoodieWriteConfig.newBuilder()
@@ -2796,7 +2793,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
assertFalse(metadataWriteConfig.isMetadataTableEnabled(), "No metadata table for metadata table");
// Metadata table should be in sync with the dataset
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClientForMetadataTable();
// Metadata table is MOR
assertEquals(metadataMetaClient.getTableType(), HoodieTableType.MERGE_ON_READ, "Metadata Table should be MOR");
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
index a987d07a22b..00b482c85fd 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/functional/TestHoodieJavaClientOnCopyOnWriteStorage.java
@@ -536,7 +536,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
final HoodieWriteConfig cfg = hoodieWriteConfig;
final String instantTime = "007";
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
String basePathStr = basePath;
HoodieTable table = getHoodieTable(metaClient, cfg);
String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
@@ -847,7 +847,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
assertNoWriteErrors(statuses);
client.commit(commitTime1, statuses);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
if (scheduleInlineClustering) {
@@ -1126,8 +1126,8 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
addConfigsForPopulateMetaFields(cfgBuilder, true);
- try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build());) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ try (HoodieJavaWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
+ HoodieTableMetaClient metaClient = createMetaClient();
HoodieJavaTable table = HoodieJavaTable.create(cfgBuilder.build(), context, metaClient);
String instantTime = "000";
@@ -1233,7 +1233,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
String instantTime = "000";
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
.withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
@@ -1262,7 +1262,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
boolean populateMetaFields) throws Exception {
String instantTime = "00000000000010";
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
Properties properties = new Properties();
if (!populateMetaFields) {
@@ -1342,7 +1342,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
100, dataGen::generateInserts, HoodieJavaWriteClient::bulkInsert, false, 100, 300,
0, true);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
@@ -1482,7 +1482,7 @@ public class TestHoodieJavaClientOnCopyOnWriteStorage extends HoodieJavaClientTe
"400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
HoodieJavaWriteClient::bulkInsert, false, 100, 100, 0, true));
commit3.get();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
index a469861c8a9..828b779be9e 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/HoodieJavaClientTestHarness.java
@@ -68,10 +68,10 @@ import org.apache.hudi.metadata.HoodieTableMetadata;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.metadata.JavaHoodieBackedTableMetadataWriter;
import org.apache.hudi.metadata.MetadataPartitionType;
-import org.apache.hudi.storage.StoragePathInfo;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
+import org.apache.hudi.storage.StoragePath;
+import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.table.HoodieJavaTable;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.utils.HoodieWriterClientTestHarness;
@@ -396,7 +396,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
HoodieWriteConfig metadataWriteConfig = metadataWriter.getWriteConfig();
assertFalse(metadataWriteConfig.isMetadataTableEnabled(), "No metadata table for metadata table");
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(hadoopConf, metadataTableBasePath);
// Metadata table is MOR
assertEquals(metadataMetaClient.getTableType(), HoodieTableType.MERGE_ON_READ, "Metadata Table should be MOR");
@@ -694,7 +694,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
assertPartitionMetadataForRecords(basePath, records, storage);
// verify that there is a commit
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
HoodieTimeline timeline = metaClient.getCommitsTimeline();
if (assertForCommit) {
@@ -785,7 +785,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
return (commit, numRecords) -> {
final HoodieIndex index = JavaHoodieIndexFactory.createIndex(writeConfig);
List<HoodieRecord> records = recordsGenFunction.apply(commit, numRecords);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieJavaTable table = HoodieJavaTable.create(writeConfig, context, metaClient);
return tagLocation(index, context, records, table);
};
@@ -809,7 +809,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
return (commit, numRecords, partition) -> {
final HoodieIndex index = JavaHoodieIndexFactory.createIndex(writeConfig);
List<HoodieRecord> records = recordsGenFunction.apply(commit, numRecords, partition);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieJavaTable table = HoodieJavaTable.create(writeConfig, context, metaClient);
return tagLocation(index, context, records, table);
};
@@ -850,7 +850,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
return (numRecords) -> {
final HoodieIndex index = JavaHoodieIndexFactory.createIndex(writeConfig);
List<HoodieKey> records = keyGenFunction.apply(numRecords);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieTable table = HoodieJavaTable.create(writeConfig, context, metaClient);
List<HoodieRecord> recordsToDelete = records.stream()
.map(key -> new HoodieAvroRecord(key, new EmptyHoodieRecordPayload())).collect(Collectors.toList());
@@ -869,7 +869,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
assertNoWriteErrors(result);
// verify that there is a commit
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient();
HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
if (assertForCommit) {
@@ -952,9 +952,7 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
String... paths) {
List<HoodieBaseFile> latestFiles = new ArrayList<>();
try {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf((Configuration) storage.getConf())
- .setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient((Configuration) storage.getConf(), basePath);
for (String path : paths) {
TableFileSystemView.BaseFileOnlyView fileSystemView =
new HoodieTableFileSystemView(metaClient,
@@ -1026,4 +1024,8 @@ public abstract class HoodieJavaClientTestHarness extends HoodieWriterClientTest
}
return builder;
}
+
+ protected HoodieTableMetaClient createMetaClient() {
+ return HoodieTestUtils.createMetaClient(hadoopConf, basePath);
+ }
}
diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
index ab446f608dc..85008bc64d9 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/testutils/TestHoodieMetadataBase.java
@@ -29,6 +29,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieCompactionConfig;
@@ -314,4 +315,8 @@ public class TestHoodieMetadataBase extends HoodieJavaClientTestHarness {
protected HoodieWriteConfig getMetadataWriteConfig(HoodieWriteConfig writeConfig) {
return HoodieMetadataWriteUtils.createMetadataWriteConfig(writeConfig, HoodieFailedWritesCleaningPolicy.LAZY);
}
+
+ protected HoodieTableMetaClient createMetaClientForMetadataTable() {
+ return HoodieTestUtils.createMetaClient(hadoopConf, metadataTableBasePath);
+ }
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
index a903503ffe3..fdb5ac40225 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestCompactionAdminClient.java
@@ -135,7 +135,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
int expNumRepairs) throws Exception {
List<Pair<HoodieLogFile, HoodieLogFile>> renameFiles =
validateUnSchedulePlan(client, ingestionInstant, compactionInstant, numEntriesPerInstant, expNumRepairs, true);
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
List<ValidationOpResult> result = client.validateCompactionPlan(metaClient, compactionInstant, 1);
if (expNumRepairs > 0) {
assertTrue(result.stream().anyMatch(r -> !r.isSuccess()), "Expect some failures in validation");
@@ -176,7 +176,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
* @param compactionInstant Compaction Instant
*/
private void ensureValidCompactionPlan(String compactionInstant) throws Exception {
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
// Ensure compaction-plan is good to begin with
List<ValidationOpResult> validationResults = client.validateCompactionPlan(metaClient, compactionInstant, 1);
assertFalse(validationResults.stream().anyMatch(v -> !v.isSuccess()),
@@ -234,7 +234,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
// Check suggested rename operations
List<Pair<HoodieLogFile, HoodieLogFile>> renameFiles =
client.getRenamingActionsForUnschedulingCompactionPlan(metaClient, compactionInstant, 1, Option.empty(), false);
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
// Log files belonging to file-slices created because of compaction request must be renamed
@@ -331,7 +331,7 @@ public class TestCompactionAdminClient extends HoodieClientTestBase {
// Call the main unschedule API
client.unscheduleCompactionFileId(op.getFileGroupId(), false, false);
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
final HoodieTableFileSystemView newFsView =
new HoodieTableFileSystemView(metaClient, metaClient.getCommitsAndCompactionTimeline());
// Expect all file-slice whose base-commit is same as compaction commit to contain no new Log files
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiFS.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiFS.java
index 8c5e6d71086..369e279ee6e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiFS.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestMultiFS.java
@@ -32,8 +32,8 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.index.HoodieIndex;
import org.apache.hudi.keygen.constant.KeyGeneratorOptions;
-import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hudi.testutils.HoodieClientTestUtils;
+import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -133,7 +133,7 @@ public class TestMultiFS extends HoodieSparkClientTestHarness {
// Read from hdfs
FileSystem fs = HadoopFSUtils.getFs(dfsBasePath, HoodieTestUtils.getDefaultHadoopConf());
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(dfsBasePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(fs.getConf(), dfsBasePath);
HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
Dataset<Row> readRecords = HoodieClientTestUtils.readCommit(dfsBasePath, sqlContext, timeline, readCommitTime);
assertEquals(readRecords.count(), records.size());
@@ -154,7 +154,7 @@ public class TestMultiFS extends HoodieSparkClientTestHarness {
LOG.info("Reading from path: " + tablePath);
fs = HadoopFSUtils.getFs(tablePath, HoodieTestUtils.getDefaultHadoopConf());
- metaClient = HoodieTableMetaClient.builder().setConf(fs.getConf()).setBasePath(tablePath).build();
+ metaClient = HoodieTestUtils.createMetaClient(fs.getConf(), tablePath);
timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
Dataset<Row> localReadRecords =
HoodieClientTestUtils.readCommit(tablePath, sqlContext, timeline, writeCommitTime);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
index a5d62a95009..0deee3abf75 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedMetadata.java
@@ -70,6 +70,7 @@ import org.apache.hudi.common.testutils.FileCreateUtils;
import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.ClosableIterator;
@@ -568,7 +569,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
}
// The earliest deltacommit in the metadata table should be "0000001",
// and the "00000000000000" init deltacommit should be archived.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
HoodieActiveTimeline metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals("0000001", metadataTimeline.getCommitsTimeline().firstInstant().get().getTimestamp());
@@ -580,7 +581,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
getHoodieWriteClient(writeConfig);
// Trigger a regular write operation. data set timeline archival should kick in.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
- archiveDataTable(writeConfig, HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build());
+ archiveDataTable(writeConfig, createMetaClient(basePath));
assertEquals("0000004",
metaClient.reloadActiveTimeline().getCommitsTimeline().firstInstant().get().getTimestamp());
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
@@ -619,13 +620,13 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// The earliest deltacommit in the metadata table should be "0000001",
// and the "00000000000000" init deltacommit should be archived.
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
HoodieActiveTimeline metadataTimeline = metadataMetaClient.reloadActiveTimeline();
assertEquals("0000001", metadataTimeline.getCommitsTimeline().firstInstant().get().getTimestamp());
getHoodieWriteClient(writeConfig);
// Trigger data table archive, should archive "0000001", "0000002"
- archiveDataTable(writeConfig, HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build());
+ archiveDataTable(writeConfig, createMetaClient(basePath));
// Trigger a regular write operation. metadata timeline archival should kick in and catch up with data table.
doWriteOperation(testTable, "000000" + (commitTime.getAndIncrement()), INSERT);
metadataTimeline = metadataMetaClient.reloadActiveTimeline();
@@ -763,7 +764,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNotNull(metadataWriter, "MetadataWriter should have been initialized");
metadataWriter.deletePartitions("0000003", Arrays.asList(COLUMN_STATS));
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
List<String> metadataTablePartitions = FSUtils.getAllPartitionPaths(engineContext, metadataMetaClient.getBasePath(), false, false);
// partition should be physically deleted
assertEquals(metadataWriter.getEnabledPartitionTypes().size(), metadataTablePartitions.size());
@@ -814,7 +815,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertTrue(tableMetadata.getLatestCompactionTime().isPresent());
assertEquals(tableMetadata.getLatestCompactionTime().get(), "0000003001");
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
HoodieWriteConfig metadataTableWriteConfig = getMetadataWriteConfig(writeConfig);
metadataMetaClient.reloadActiveTimeline();
@@ -1075,14 +1076,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
private void revertTableToInflightState(HoodieWriteConfig writeConfig) throws IOException {
String basePath = writeConfig.getBasePath();
String mdtBasePath = getMetadataTableBasePath(basePath);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(new Configuration())
- .setBasePath(basePath)
- .build();
- HoodieTableMetaClient mdtMetaClient = HoodieTableMetaClient.builder()
- .setConf(new Configuration())
- .setBasePath(mdtBasePath)
- .build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
+ HoodieTableMetaClient mdtMetaClient = createMetaClient(mdtBasePath);
HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
HoodieActiveTimeline mdtTimeline = mdtMetaClient.getActiveTimeline();
assertEquals(1, timeline.countInstants());
@@ -1173,10 +1168,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// 2nd commit
doWriteOperation(testTable, "0000001", INSERT);
- final HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
- .setConf(hadoopConf)
- .setBasePath(metadataTableBasePath)
- .build();
+ final HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
HoodieWriteConfig metadataTableWriteConfig = getMetadataWriteConfig(writeConfig);
metadataMetaClient.reloadActiveTimeline();
final HoodieTable table = HoodieSparkTable.create(metadataTableWriteConfig, context, metadataMetaClient);
@@ -1962,9 +1954,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
assertNoWriteErrors(writeStatuses);
// ensure that 000003 is after rollback of the partially failed 2nd commit.
- HoodieTableMetaClient metadataMetaClient =
- HoodieTableMetaClient.builder().setBasePath(metaClient.getMetaPath() + "/metadata/")
- .setConf(metaClient.getHadoopConf()).build();
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(
+ metaClient.getHadoopConf(), metaClient.getMetaPath() + "/metadata/");
HoodieInstant rollbackInstant =
metadataMetaClient.getActiveTimeline().getRollbackTimeline().getInstants().get(0);
@@ -2137,7 +2128,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
executors.shutdown();
// Ensure all commits were synced to the Metadata Table
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
assertEquals(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline().filterCompletedInstants().countInstants(), 5);
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "0000002")));
assertTrue(metadataMetaClient.getActiveTimeline().containsInstant(new HoodieInstant(false, HoodieTimeline.DELTA_COMMIT_ACTION, "0000003")));
@@ -2187,7 +2178,7 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
// Ensure all commits were synced to the Metadata Table
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
LOG.warn("total commits in metadata table " + metadataMetaClient.getActiveTimeline().getCommitsTimeline().countInstants());
// 6 commits and 2 cleaner commits.
@@ -2451,8 +2442,8 @@ public class TestHoodieBackedMetadata extends TestHoodieMetadataBase {
client.insert(jsc.parallelize(records, 1), newCommitTime).collect();
}
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
- HoodieTableMetaClient datasetMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(config.getBasePath()).build();
+ HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
+ HoodieTableMetaClient datasetMetaClient = createMetaClient(config.getBasePath());
// There should not be any compaction yet and we have not performed more than maxDeltaCommitsBeforeCompaction
// deltacommits (1 will be due to bootstrap)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
index de1148f29ea..61f7ea5323d 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieBackedTableMetadata.java
@@ -255,10 +255,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
// 2nd commit
doWriteOperation(testTable, "0000001", INSERT);
- final HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
- .setConf(hadoopConf)
- .setBasePath(metadataTableBasePath)
- .build();
+ final HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
HoodieWriteConfig metadataTableWriteConfig = getMetadataWriteConfig(writeConfig);
metadataMetaClient.reloadActiveTimeline();
final HoodieTable table = HoodieSparkTable.create(metadataTableWriteConfig, context, metadataMetaClient);
@@ -328,10 +325,7 @@ public class TestHoodieBackedTableMetadata extends TestHoodieMetadataBase {
HoodieCommitMetadata commitMetadata2 =
testTable.doWriteOperation(instant2, BULK_INSERT, emptyList(), asList(partition), 1);
- final HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
- .setConf(hadoopConf)
- .setBasePath(metadataTableBasePath)
- .build();
+ final HoodieTableMetaClient metadataMetaClient = createMetaClient(metadataTableBasePath);
while (getNumCompactions(metadataMetaClient) == 0) {
// Write until the compaction happens in the metadata table
testTable.doWriteOperation(
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
index c6f04c83998..643a68762a0 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
@@ -701,7 +701,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
final HoodieWriteConfig cfg = hoodieWriteConfig;
final String instantTime = "007";
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieClientTestUtils.createMetaClient(jsc, basePath);
String basePathStr = basePath;
HoodieTable table = getHoodieTable(metaClient, cfg);
String extension = metaClient.getTableConfig().getBaseFileFormat().getFileExtension();
@@ -1256,7 +1256,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statuses);
assertEquals(2, statuses.size(), "2 files needs to be committed.");
- HoodieTableMetaClient metadata = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metadata = createMetaClient(basePath);
HoodieTable table = getHoodieTable(metadata, config);
BaseFileOnlyView fileSystemView = table.getBaseFileOnlyView();
@@ -1365,7 +1365,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
+ fileUtils.readRowKeys(hadoopConf, new StoragePath(basePath, statuses.get(1).getStat().getPath())).size(),
"file should contain 340 records");
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
HoodieTable table = getHoodieTable(metaClient, config);
List<HoodieBaseFile> files = table.getBaseFileOnlyView()
.getLatestBaseFilesBeforeOrOn(testPartitionPath, commitTime3).collect(Collectors.toList());
@@ -1524,7 +1524,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
List<WriteStatus> statusList = statuses.collect();
assertNoWriteErrors(statusList);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
assertEquals(2, metaClient.getActiveTimeline().getCommitsTimeline().filterInflightsAndRequested().countInstants());
// trigger another commit. this should rollback latest partial commit.
@@ -1564,7 +1564,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
assertNoWriteErrors(statusList);
client.commit(commitTime1, statuses);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
if (scheduleInlineClustering) {
@@ -1611,7 +1611,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// start clustering, but don't commit
List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, populateMetaFields, false);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
assertEquals(1, pendingClusteringPlans.size());
@@ -1673,7 +1673,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
// start clustering, but don't commit keep it inflight
List<HoodieRecord> allRecords = testInsertAndClustering(clusteringConfig, true, false);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
List<Pair<HoodieInstant, HoodieClusteringPlan>> pendingClusteringPlans =
ClusteringUtils.getAllPendingClusteringPlans(metaClient).collect(Collectors.toList());
assertEquals(1, pendingClusteringPlans.size());
@@ -2224,7 +2224,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
HoodieWriteConfig.Builder cfgBuilder = getConfigBuilder().withAutoCommit(false);
addConfigsForPopulateMetaFields(cfgBuilder, true);
try (SparkRDDWriteClient client = getHoodieWriteClient(cfgBuilder.build())) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
HoodieSparkTable table = HoodieSparkTable.create(cfgBuilder.build(), context, metaClient);
String instantTime = "000";
@@ -2334,7 +2334,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
@ParameterizedTest
@ValueSource(booleans = {true, false})
public void testConsistencyCheckDuringFinalize(boolean enableOptimisticConsistencyGuard) throws Exception {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
String instantTime = "000";
HoodieWriteConfig cfg = getConfigBuilder().withAutoCommit(false).withConsistencyGuardConfig(ConsistencyGuardConfig.newBuilder()
.withEnableOptimisticConsistencyGuard(enableOptimisticConsistencyGuard).build()).build();
@@ -2364,7 +2364,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
private void testRollbackAfterConsistencyCheckFailureUsingFileList(boolean rollbackUsingMarkers, boolean enableOptimisticConsistencyGuard,
boolean populateMetaFields) throws Exception {
String instantTime = "00000000000010";
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
Properties properties = new Properties();
if (!populateMetaFields) {
@@ -2448,7 +2448,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
writeBatch(client, "400", "300", Option.of(Arrays.asList("400")), "400",
100, dataGen::generateInserts, SparkRDDWriteClient::bulkInsert, false, 100, 300,
0, true);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
@@ -2588,7 +2588,7 @@ public class TestHoodieClientOnCopyOnWriteStorage extends HoodieClientTestBase {
"400", "300", Option.of(Arrays.asList("400")), "300", 100, dataGen::generateInserts,
SparkRDDWriteClient::bulkInsert, false, 100, 100, 0, true));
commit3.get();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
assertTrue(metaClient.getActiveTimeline().getTimelineOfActions(
CollectionUtils.createSet(ROLLBACK_ACTION)).countInstants() == 0);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
index e867ec3cd5f..9aae0a60ec8 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestRemoteFileSystemViewWithMetadataTable.java
@@ -174,10 +174,7 @@ public class TestRemoteFileSystemViewWithMetadataTable extends HoodieSparkClient
private void runAssertionsForBasePath(boolean useExistingTimelineServer, String basePathStr, SparkRDDWriteClient writeClient) throws IOException {
// At this point, there are three deltacommits and one compaction commit in the Hudi timeline,
// and the file system view of timeline server is not yet synced
- HoodieTableMetaClient newMetaClient = HoodieTableMetaClient.builder()
- .setConf(hadoopConf)
- .setBasePath(basePathStr)
- .build();
+ HoodieTableMetaClient newMetaClient = createMetaClient(basePathStr);
HoodieActiveTimeline timeline = newMetaClient.getActiveTimeline();
HoodieInstant compactionCommit = timeline.lastInstant().get();
assertTrue(timeline.lastInstant().get().getAction().equals(COMMIT_ACTION));
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
index d9ef683b2b6..0ca22e5f226 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/CompactionTestBase.java
@@ -89,7 +89,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
**/
protected void validateDeltaCommit(String latestDeltaCommit, final Map<HoodieFileGroupId, Pair<String, HoodieCompactionOperation>> fgIdToCompactionOperation,
HoodieWriteConfig cfg) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
HoodieTable table = getHoodieTable(metaClient, cfg);
List<FileSlice> fileSliceList = getCurrentLatestFileSlices(table);
fileSliceList.forEach(fileSlice -> {
@@ -110,7 +110,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
List<HoodieRecord> records, HoodieWriteConfig cfg, boolean insertFirst, List<String> expPendingCompactionInstants)
throws Exception {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
List<Pair<String, HoodieCompactionPlan>> pendingCompactions = readClient.getPendingCompactions();
List<String> gotPendingCompactionInstants =
pendingCompactions.stream().map(pc -> pc.getKey()).sorted().collect(Collectors.toList());
@@ -132,7 +132,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
client.commit(firstInstant, statuses);
}
assertNoWriteErrors(statusList);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
HoodieTable hoodieTable = getHoodieTable(metaClient, cfg);
List<HoodieBaseFile> dataFilesToRead = getCurrentLatestBaseFiles(hoodieTable);
assertTrue(dataFilesToRead.stream().findAny().isPresent(),
@@ -143,7 +143,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
int numRecords = records.size();
for (String instantTime : deltaInstants) {
records = dataGen.generateUpdates(instantTime, numRecords);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
createNextDeltaCommit(instantTime, records, client, metaClient, cfg, false);
validateDeltaCommit(instantTime, fgIdToCompactionOperation, cfg);
}
@@ -151,7 +151,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
}
protected void moveCompactionFromRequestedToInflight(String compactionInstantTime, HoodieWriteConfig cfg) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
HoodieInstant compactionInstant = HoodieTimeline.getCompactionRequestedInstant(compactionInstantTime);
metaClient.getActiveTimeline().transitionCompactionRequestedToInflight(compactionInstant);
HoodieInstant instant = metaClient.getActiveTimeline().reload().filterPendingCompactionTimeline().getInstantsAsStream()
@@ -161,7 +161,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
protected void scheduleCompaction(String compactionInstantTime, SparkRDDWriteClient client, HoodieWriteConfig cfg) {
client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty());
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
HoodieInstant instant = metaClient.getActiveTimeline().filterPendingCompactionTimeline().lastInstant().get();
assertEquals(compactionInstantTime, instant.getTimestamp(), "Last compaction instant must be the one set");
}
@@ -177,7 +177,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
*/
protected String tryScheduleCompaction(String compactionInstantTime, SparkRDDWriteClient client, HoodieWriteConfig cfg) {
client.scheduleCompactionAtInstant(compactionInstantTime, Option.empty());
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
return metaClient.getActiveTimeline().filterPendingCompactionTimeline().lastInstant().map(HoodieInstant::getTimestamp).orElse(null);
}
@@ -209,7 +209,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
}
// verify that there is a commit
- table = getHoodieTable(HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).setLoadActiveTimelineOnLoad(true).build(), cfg);
+ table = getHoodieTable(createMetaClient(cfg.getBasePath()), cfg);
HoodieTimeline timeline = table.getMetaClient().getCommitTimeline().filterCompletedInstants();
String latestCompactionCommitTime = timeline.lastInstant().get().getTimestamp();
assertEquals(latestCompactionCommitTime, compactionInstantTime,
@@ -231,7 +231,7 @@ public class CompactionTestBase extends HoodieClientTestBase {
"Compacted files should not show up in latest slices");
// verify that there is a commit
- table = getHoodieTable(HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).setLoadActiveTimelineOnLoad(true).build(), cfg);
+ table = getHoodieTable(createMetaClient(cfg.getBasePath()), cfg);
HoodieTimeline timeline = table.getMetaClient().getCommitTimeline().filterCompletedInstants();
// verify compaction commit is visible in timeline
assertTrue(timeline.filterCompletedInstants().getInstantsAsStream()
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
index 3ab6580e72b..209d70e499a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/compact/TestInlineCompaction.java
@@ -76,7 +76,7 @@ public class TestInlineCompaction extends CompactionTestBase {
SparkRDDReadClient readClient = getHoodieReadClient(cfg.getBasePath());
List<String> instants = IntStream.range(0, 2).mapToObj(i -> HoodieActiveTimeline.createNewInstantTime()).collect(Collectors.toList());
runNextDeltaCommits(writeClient, readClient, instants, records, cfg, true, new ArrayList<>());
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
// Then: ensure no compaction is executed since there are only 2 delta commits
assertEquals(2, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
@@ -95,12 +95,12 @@ public class TestInlineCompaction extends CompactionTestBase {
runNextDeltaCommits(writeClient, readClient, instants, records, cfg, true, new ArrayList<>());
// third commit, that will trigger compaction
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
String finalInstant = HoodieActiveTimeline.createNewInstantTime();
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 100), writeClient, metaClient, cfg, false);
// Then: ensure the file slices are compacted as per policy
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
assertEquals(HoodieTimeline.COMMIT_ACTION, metaClient.getActiveTimeline().lastInstant().get().getAction());
String compactionTime = metaClient.getActiveTimeline().lastInstant().get().getTimestamp();
@@ -125,10 +125,10 @@ public class TestInlineCompaction extends CompactionTestBase {
String requestInstant = HoodieActiveTimeline.createNewInstantTime();
scheduleCompaction(requestInstant, writeClient, cfg);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(metaClient.getActiveTimeline().getInstantsAsStream()
- .filter(hoodieInstant -> hoodieInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)
- && hoodieInstant.getState() == HoodieInstant.State.REQUESTED).count(), 1);
+ .filter(hoodieInstant -> hoodieInstant.getAction().equals(HoodieTimeline.COMPACTION_ACTION)
+ && hoodieInstant.getState() == HoodieInstant.State.REQUESTED).count(), 1);
// step 2: try to create another, but this one should fail because the NUM_COMMITS_AFTER_LAST_REQUEST strategy ,
// and will throw a AssertionError due to scheduleCompaction will check if the last instant is a compaction request
@@ -157,7 +157,7 @@ public class TestInlineCompaction extends CompactionTestBase {
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 100), newWriteClient, metaClient, cfg, false);
}
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
// step 5: there should be only 2 .commit, and no pending compaction.
// the last instant should be delta commit since the compaction request is earlier.
assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().filter(instant -> instant.getAction().equals(HoodieTimeline.COMMIT_ACTION))
@@ -180,11 +180,11 @@ public class TestInlineCompaction extends CompactionTestBase {
// after 10s, that will trigger compaction
String finalInstant = HoodieActiveTimeline.createNewInstantTime(10000);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 100), writeClient, metaClient, cfg, false);
// Then: ensure the file slices are compacted as per policy
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(3, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
assertEquals(HoodieTimeline.COMMIT_ACTION, metaClient.getActiveTimeline().lastInstant().get().getAction());
}
@@ -201,17 +201,17 @@ public class TestInlineCompaction extends CompactionTestBase {
runNextDeltaCommits(writeClient, readClient, instants, records, cfg, true, new ArrayList<>());
// Then: trigger the compaction because reach 3 commits.
String finalInstant = HoodieActiveTimeline.createNewInstantTime();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 10), writeClient, metaClient, cfg, false);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
// 4th commit, that will trigger compaction because reach the time elapsed
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
finalInstant = HoodieActiveTimeline.createNewInstantTime(60000);
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 10), writeClient, metaClient, cfg, false);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(6, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
}
}
@@ -225,16 +225,16 @@ public class TestInlineCompaction extends CompactionTestBase {
SparkRDDReadClient readClient = getHoodieReadClient(cfg.getBasePath());
List<String> instants = IntStream.range(0, 2).mapToObj(i -> HoodieActiveTimeline.createNewInstantTime()).collect(Collectors.toList());
runNextDeltaCommits(writeClient, readClient, instants, records, cfg, true, new ArrayList<>());
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
// Then: ensure no compaction is executed since there are only 3 delta commits
assertEquals(2, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
// 3d commit, that will trigger compaction
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
String finalInstant = HoodieActiveTimeline.createNewInstantTime(20000);
createNextDeltaCommit(finalInstant, dataGen.generateUpdates(finalInstant, 10), writeClient, metaClient, cfg, false);
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
}
}
@@ -263,12 +263,12 @@ public class TestInlineCompaction extends CompactionTestBase {
HoodieWriteConfig inlineCfg = getConfigForInlineCompaction(2, 60, CompactionTriggerStrategy.NUM_COMMITS);
String instantTime3 = HoodieActiveTimeline.createNewInstantTime();
try (SparkRDDWriteClient<?> writeClient = getHoodieWriteClient(inlineCfg)) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
createNextDeltaCommit(instantTime3, dataGen.generateUpdates(instantTime3, 100), writeClient, metaClient, inlineCfg, false);
}
// Then: 1 delta commit is done, the failed compaction is retried
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
assertEquals(instantTime2, metaClient.getActiveTimeline().getCommitTimeline().filterCompletedInstants().firstInstant().get().getTimestamp());
}
@@ -299,13 +299,13 @@ public class TestInlineCompaction extends CompactionTestBase {
HoodieWriteConfig inlineCfg = getConfigForInlineCompaction(5, 1000, CompactionTriggerStrategy.TIME_ELAPSED);
String instantTime2;
try (SparkRDDWriteClient<?> writeClient = getHoodieWriteClient(inlineCfg)) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
instantTime2 = HoodieActiveTimeline.createNewInstantTime();
createNextDeltaCommit(instantTime2, dataGen.generateUpdates(instantTime2, 10), writeClient, metaClient, inlineCfg, false);
}
// Then: 1 delta commit is done, the failed compaction is retried
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
// 2 delta commits at the beginning. 1 compaction, 1 delta commit following it.
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
assertEquals(instantTime, metaClient.getActiveTimeline().getCommitTimeline().filterCompletedInstants().firstInstant().get().getTimestamp());
@@ -337,13 +337,13 @@ public class TestInlineCompaction extends CompactionTestBase {
HoodieWriteConfig inlineCfg = getConfigForInlineCompaction(3, 20, CompactionTriggerStrategy.NUM_OR_TIME);
String instantTime2;
try (SparkRDDWriteClient<?> writeClient = getHoodieWriteClient(inlineCfg)) {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.getBasePath());
instantTime2 = HoodieActiveTimeline.createNewInstantTime();
createNextDeltaCommit(instantTime2, dataGen.generateUpdates(instantTime2, 10), writeClient, metaClient, inlineCfg, false);
}
// Then: 1 delta commit is done, the failed compaction is retried
- metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(cfg.getBasePath()).build();
+ metaClient = createMetaClient(cfg.getBasePath());
assertEquals(4, metaClient.getActiveTimeline().getWriteTimeline().countInstants());
assertEquals(instantTime, metaClient.getActiveTimeline().getCommitTimeline().filterCompletedInstants().firstInstant().get().getTimestamp());
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
index 1cfb6704ab3..34bf3f66d3f 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieCleanerTestBase.java
@@ -130,10 +130,9 @@ public class HoodieCleanerTestBase extends HoodieClientTestBase {
if (config.isMetadataTableEnabled() && simulateMetadataFailure) {
// Simulate the failure of corresponding instant in the metadata table
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
- .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()))
- .setConf(metaClient.getHadoopConf())
- .build();
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(
+ metaClient.getHadoopConf(),
+ HoodieTableMetadata.getMetadataTableBasePath(metaClient.getBasePath()));
HoodieInstant deltaCommit = new HoodieInstant(HoodieInstant.State.COMPLETED, HoodieTimeline.DELTA_COMMIT_ACTION, cleanInstantTs);
metadataMetaClient.reloadActiveTimeline().revertToInflight(deltaCommit);
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
index b11d53d9454..95ee7e0544b 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestBase.java
@@ -31,6 +31,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.view.SyncableFileSystemView;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex;
@@ -107,7 +108,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
return (commit, numRecords) -> {
final HoodieIndex index = SparkHoodieIndexFactory.createIndex(writeConfig);
List<HoodieRecord> records = recordsGenFunction.apply(commit, numRecords);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieSparkTable table = HoodieSparkTable.create(writeConfig, context, metaClient);
JavaRDD<HoodieRecord> taggedRecords = tagLocation(index, context, context.getJavaSparkContext().parallelize(records, 1), table);
return taggedRecords.collect();
@@ -132,7 +133,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
return (commit, numRecords, partition) -> {
final HoodieIndex index = SparkHoodieIndexFactory.createIndex(writeConfig);
List<HoodieRecord> records = recordsGenFunction.apply(commit, numRecords, partition);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieSparkTable table = HoodieSparkTable.create(writeConfig, context, metaClient);
JavaRDD<HoodieRecord> taggedRecords = tagLocation(index, context, context.getJavaSparkContext().parallelize(records, 1), table);
return taggedRecords.collect();
@@ -157,7 +158,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
return (numRecords) -> {
final HoodieIndex index = SparkHoodieIndexFactory.createIndex(writeConfig);
List<HoodieKey> records = keyGenFunction.apply(numRecords);
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieSparkTable table = HoodieSparkTable.create(writeConfig, context, metaClient);
JavaRDD<HoodieRecord> recordsToDelete = context.getJavaSparkContext().parallelize(records, 1)
.map(key -> new HoodieAvroRecord(key, new EmptyHoodieRecordPayload()));
@@ -475,7 +476,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
assertPartitionMetadataForRecords(basePath, records, storage);
// verify that there is a commit
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieTimeline timeline = metaClient.getCommitsTimeline();
if (assertForCommit) {
@@ -527,7 +528,7 @@ public class HoodieClientTestBase extends HoodieSparkClientTestHarness {
assertNoWriteErrors(statuses);
// verify that there is a commit
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hadoopConf, basePath);
HoodieTimeline timeline = new HoodieActiveTimeline(metaClient).getCommitTimeline();
if (assertForCommit) {
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
index 0ffe94e754c..784dbd764a0 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieClientTestUtils.java
@@ -34,13 +34,14 @@ import org.apache.hudi.common.table.view.FileSystemViewManager;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.HoodieTableFileSystemView;
import org.apache.hudi.common.table.view.TableFileSystemView.BaseFileOnlyView;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
+import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.timeline.service.TimelineService;
import org.apache.avro.generic.GenericRecord;
@@ -51,6 +52,7 @@ import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -226,11 +228,7 @@ public class HoodieClientTestUtils {
String... paths) {
List<HoodieBaseFile> latestFiles = new ArrayList<>();
try {
- HoodieTableMetaClient metaClient =
- HoodieTableMetaClient.builder()
- .setConf((Configuration) storage.getConf())
- .setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(storage, basePath);
for (String path : paths) {
BaseFileOnlyView fileSystemView = new HoodieTableFileSystemView(
metaClient,
@@ -309,6 +307,24 @@ public class HoodieClientTestUtils {
}
}
+ /**
+ * @param jsc {@link JavaSparkContext} instance.
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance.
+ */
+ public static HoodieTableMetaClient createMetaClient(JavaSparkContext jsc, String basePath) {
+ return HoodieTestUtils.createMetaClient(jsc.hadoopConfiguration(), basePath);
+ }
+
+ /**
+ * @param spark {@link SparkSession} instance.
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance.
+ */
+ public static HoodieTableMetaClient createMetaClient(SparkSession spark, String basePath) {
+ return HoodieTestUtils.createMetaClient(spark.sessionState().newHadoopConf(), basePath);
+ }
+
private static Option<HoodieCommitMetadata> getCommitMetadataForInstant(HoodieTableMetaClient metaClient, HoodieInstant instant) {
try {
HoodieTimeline timeline = metaClient.getActiveTimeline().getCommitsTimeline().filterCompletedInstants();
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
index fe977aba877..fc30981a1ac 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/HoodieSparkClientTestHarness.java
@@ -406,7 +406,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
}
public HoodieTableMetaClient getHoodieMetaClient(Configuration conf, String basePath) {
- metaClient = HoodieTableMetaClient.builder().setConf(conf).setBasePath(basePath).build();
+ metaClient = HoodieTestUtils.createMetaClient(conf, basePath);
return metaClient;
}
@@ -619,7 +619,7 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
HoodieWriteConfig metadataWriteConfig = metadataWriter.getWriteConfig();
assertFalse(metadataWriteConfig.isMetadataTableEnabled(), "No metadata table for metadata table");
- HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder().setConf(hadoopConf).setBasePath(metadataTableBasePath).build();
+ HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient(hadoopConf, metadataTableBasePath);
// Metadata table is MOR
assertEquals(metadataMetaClient.getTableType(), HoodieTableType.MERGE_ON_READ, "Metadata Table should be MOR");
@@ -684,4 +684,16 @@ public abstract class HoodieSparkClientTestHarness extends HoodieWriterClientTes
}
return new HoodieInstant(inflightOnly, "clean", instantTime);
}
+
+ protected HoodieTableMetaClient createMetaClient(String basePath) {
+ return HoodieTestUtils.createMetaClient(hadoopConf, basePath);
+ }
+
+ protected HoodieTableMetaClient createMetaClient(SparkSession spark, String basePath) {
+ return HoodieClientTestUtils.createMetaClient(spark, basePath);
+ }
+
+ protected HoodieTableMetaClient createMetaClient(JavaSparkContext context, String basePath) {
+ return HoodieClientTestUtils.createMetaClient(context, basePath);
+ }
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
index 36fea5c83a1..eca9162af77 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/FileCreateUtils.java
@@ -521,9 +521,8 @@ public class FileCreateUtils {
String... paths) {
Map<String, Long> toReturn = new HashMap<>();
try {
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf((Configuration) storage.getConf()).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ (Configuration) storage.getConf(), basePath);
for (String path : paths) {
TableFileSystemView.BaseFileOnlyView fileSystemView =
new HoodieTableFileSystemView(metaClient,
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
index bda5b38c517..e5096cc1036 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieCommonTestHarness.java
@@ -113,7 +113,7 @@ public class HoodieCommonTestHarness {
}
protected void refreshFsView() throws IOException {
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
}
protected SyncableFileSystemView getFileSystemView(HoodieTimeline timeline) throws IOException {
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index 46a006aae7e..8713b76bb6d 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -18,6 +18,7 @@
package org.apache.hudi.common.testutils;
+import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.model.HoodieAvroPayload;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieTableType;
@@ -26,6 +27,7 @@ import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.storage.HoodieStorage;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
@@ -178,6 +180,45 @@ public class HoodieTestUtils {
return init(getDefaultHadoopConf(), basePath, tableType, props);
}
+ /**
+ * @param conf file system configuration.
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance.
+ */
+ public static HoodieTableMetaClient createMetaClient(Configuration conf,
+ String basePath) {
+ return HoodieTableMetaClient.builder()
+ .setConf(conf).setBasePath(basePath).build();
+ }
+
+ /**
+ * @param storage {@link HoodieStorage} instance.
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance.
+ */
+ public static HoodieTableMetaClient createMetaClient(HoodieStorage storage,
+ String basePath) {
+ return createMetaClient((Configuration) storage.getConf(), basePath);
+ }
+
+ /**
+ * @param context Hudi engine context.
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance.
+ */
+ public static HoodieTableMetaClient createMetaClient(HoodieEngineContext context,
+ String basePath) {
+ return createMetaClient(context.getHadoopConf().get(), basePath);
+ }
+
+ /**
+ * @param basePath base path of the Hudi table.
+ * @return a new {@link HoodieTableMetaClient} instance with default configuration for tests.
+ */
+ public static HoodieTableMetaClient createMetaClient(String basePath) {
+ return createMetaClient(getDefaultHadoopConf(), basePath);
+ }
+
public static <T extends Serializable> T serializeDeserialize(T object, Class<T> clazz) {
// Using Kryo as the default serializer in Spark Jobs
Kryo kryo = new Kryo();
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
index 546559b674c..844d038a27b 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/util/TestCompactionUtils.java
@@ -33,6 +33,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.versioning.compaction.CompactionPlanMigrator;
import org.apache.hudi.common.testutils.CompactionTestUtils.DummyHoodieBaseFile;
import org.apache.hudi.common.testutils.HoodieCommonTestHarness;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.storage.StoragePath;
@@ -216,7 +217,7 @@ public class TestCompactionUtils extends HoodieCommonTestHarness {
// schedule similar plan again so that there will be duplicates
plan1.getOperations().get(0).setDataFilePath("bla");
scheduleCompaction(metaClient, "005", plan1);
- metaClient = HoodieTableMetaClient.builder().setConf(metaClient.getHadoopConf()).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build();
+ metaClient = HoodieTestUtils.createMetaClient(metaClient.getHadoopConf(), basePath);
assertThrows(IllegalStateException.class, () -> {
CompactionUtils.getAllPendingCompactionOperations(metaClient);
});
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
index 9ab3ceb0461..c612d1f1365 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.config.HoodieCleanConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.configuration.FlinkOptions;
@@ -37,7 +38,6 @@ import org.apache.hudi.sink.event.WriteMetadataEvent;
import org.apache.hudi.sink.utils.MockCoordinatorExecutor;
import org.apache.hudi.sink.utils.NonThrownExecutor;
import org.apache.hudi.storage.HoodieStorage;
-import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestUtils;
@@ -314,7 +314,7 @@ public class TestStreamWriteOperatorCoordinator {
assertNotEquals("", instant);
final String metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(tempFile.getAbsolutePath());
- HoodieTableMetaClient metadataTableMetaClient = StreamerUtil.createMetaClient(metadataTableBasePath, HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metadataTableMetaClient = HoodieTestUtils.createMetaClient(HadoopConfigurations.getHadoopConf(conf), metadataTableBasePath);
HoodieTimeline completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
assertThat("One instant need to sync to metadata table", completedTimeline.countInstants(), is(1));
assertThat(completedTimeline.lastInstant().get().getTimestamp(), startsWith(HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP));
@@ -386,7 +386,7 @@ public class TestStreamWriteOperatorCoordinator {
assertNotEquals("", instant);
final String metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(tempFile.getAbsolutePath());
- HoodieTableMetaClient metadataTableMetaClient = StreamerUtil.createMetaClient(metadataTableBasePath, HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metadataTableMetaClient = HoodieTestUtils.createMetaClient(HadoopConfigurations.getHadoopConf(conf), metadataTableBasePath);
HoodieTimeline completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
assertThat("One instant need to sync to metadata table", completedTimeline.countInstants(), is(1));
assertThat(completedTimeline.lastInstant().get().getTimestamp(), startsWith(HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP));
@@ -431,7 +431,7 @@ public class TestStreamWriteOperatorCoordinator {
assertNotEquals("", instant);
final String metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(tempFile.getAbsolutePath());
- HoodieTableMetaClient metadataTableMetaClient = StreamerUtil.createMetaClient(metadataTableBasePath, HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metadataTableMetaClient = HoodieTestUtils.createMetaClient(HadoopConfigurations.getHadoopConf(conf), metadataTableBasePath);
HoodieTimeline completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
assertThat("One instant need to sync to metadata table", completedTimeline.countInstants(), is(1));
assertThat(completedTimeline.lastInstant().get().getTimestamp(), startsWith(HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP));
@@ -514,7 +514,7 @@ public class TestStreamWriteOperatorCoordinator {
assertNotEquals("", instant);
final String metadataTableBasePath = HoodieTableMetadata.getMetadataTableBasePath(tempFile.getAbsolutePath());
- HoodieTableMetaClient metadataTableMetaClient = StreamerUtil.createMetaClient(metadataTableBasePath, HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metadataTableMetaClient = HoodieTestUtils.createMetaClient(HadoopConfigurations.getHadoopConf(conf), metadataTableBasePath);
HoodieTimeline completedTimeline = metadataTableMetaClient.getActiveTimeline().filterCompletedInstants();
assertThat("One instant need to sync to metadata table", completedTimeline.countInstants(), is(1));
assertThat(completedTimeline.lastInstant().get().getTimestamp(), startsWith(HoodieTableMetadata.SOLO_COMMIT_TIMESTAMP));
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
index e45553eba21..27a21bfab36 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/bucket/ITTestBucketStreamWrite.java
@@ -25,12 +25,12 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.testutils.FileCreateUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.index.HoodieIndex.IndexType;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.HoodieStorageUtils;
import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.util.StreamerUtil;
import org.apache.hudi.utils.FlinkMiniCluster;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestData;
@@ -95,7 +95,7 @@ public class ITTestBucketStreamWrite {
private static void doDeleteCommit(String tablePath, boolean isCow) throws Exception {
// create metaClient
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tablePath, new org.apache.hadoop.conf.Configuration());
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(tablePath);
// should only contain one instant
HoodieTimeline activeCompletedTimeline = metaClient.getActiveTimeline().filterCompletedInstants();
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java
index d883b72b075..2781e3f8153 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieCatalog.java
@@ -24,6 +24,7 @@ import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
@@ -78,6 +79,7 @@ import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient;
import static org.apache.hudi.table.catalog.CatalogOptions.CATALOG_PATH;
import static org.apache.hudi.table.catalog.CatalogOptions.DEFAULT_DATABASE;
import static org.hamcrest.CoreMatchers.instanceOf;
@@ -262,8 +264,8 @@ public class TestHoodieCatalog {
() -> catalog.createTable(tablePath, EXPECTED_CATALOG_TABLE, false));
// validate key generator for partitioned table
- HoodieTableMetaClient metaClient =
- StreamerUtil.createMetaClient(catalog.inferTablePath(catalogPathStr, tablePath), new org.apache.hadoop.conf.Configuration());
+ HoodieTableMetaClient metaClient = createMetaClient(
+ catalog.inferTablePath(catalogPathStr, tablePath));
String keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertEquals(keyGeneratorClassName, SimpleAvroKeyGenerator.class.getName());
@@ -279,8 +281,8 @@ public class TestHoodieCatalog {
);
catalog.createTable(singleKeyMultiplePartitionPath, singleKeyMultiplePartitionTable, false);
- metaClient =
- StreamerUtil.createMetaClient(catalog.inferTablePath(catalogPathStr, singleKeyMultiplePartitionPath), new org.apache.hadoop.conf.Configuration());
+ metaClient = createMetaClient(
+ catalog.inferTablePath(catalogPathStr, singleKeyMultiplePartitionPath));
keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertThat(keyGeneratorClassName, is(ComplexAvroKeyGenerator.class.getName()));
@@ -296,8 +298,8 @@ public class TestHoodieCatalog {
);
catalog.createTable(multipleKeySinglePartitionPath, multipleKeySinglePartitionTable, false);
- metaClient =
- StreamerUtil.createMetaClient(catalog.inferTablePath(catalogPathStr, singleKeyMultiplePartitionPath), new org.apache.hadoop.conf.Configuration());
+ metaClient = createMetaClient(
+ catalog.inferTablePath(catalogPathStr, singleKeyMultiplePartitionPath));
keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertThat(keyGeneratorClassName, is(ComplexAvroKeyGenerator.class.getName()));
@@ -314,8 +316,7 @@ public class TestHoodieCatalog {
catalog.createTable(nonPartitionPath, nonPartitionCatalogTable, false);
- metaClient =
- StreamerUtil.createMetaClient(catalog.inferTablePath(catalogPathStr, nonPartitionPath), new org.apache.hadoop.conf.Configuration());
+ metaClient = createMetaClient(catalog.inferTablePath(catalogPathStr, nonPartitionPath));
keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertEquals(keyGeneratorClassName, NonpartitionedAvroKeyGenerator.class.getName());
}
@@ -423,7 +424,7 @@ public class TestHoodieCatalog {
String tablePathStr = catalog.inferTablePath(catalogPathStr, tablePath);
Configuration flinkConf = TestConfigurations.getDefaultConf(tablePathStr);
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tablePathStr, HadoopConfigurations.getHadoopConf(flinkConf));
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(HadoopConfigurations.getHadoopConf(flinkConf), tablePathStr);
TestData.writeData(TestData.DATA_SET_INSERT, flinkConf);
assertTrue(catalog.partitionExists(tablePath, partitionSpec));
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
index 24621e1b8d7..76bd2857e39 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/catalog/TestHoodieHiveCatalog.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.model.HoodieTableType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.configuration.FlinkOptions;
@@ -221,8 +222,8 @@ public class TestHoodieHiveCatalog {
assertEquals("id", table2.getOptions().get(FlinkOptions.RECORD_KEY_FIELD.key()));
// validate key generator for partitioned table
- HoodieTableMetaClient metaClient =
- StreamerUtil.createMetaClient(hoodieCatalog.inferTablePath(tablePath, table), createHiveConf());
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ createHiveConf(), hoodieCatalog.inferTablePath(tablePath, table));
String keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertEquals(keyGeneratorClassName, SimpleAvroKeyGenerator.class.getName());
@@ -232,8 +233,9 @@ public class TestHoodieHiveCatalog {
new CatalogTableImpl(singleKeyMultiPartitionTableSchema, multiPartitions, options, "hudi table");
hoodieCatalog.createTable(singleKeyMultiPartitionPath, singleKeyMultiPartitionTable, false);
- HoodieTableMetaClient singleKeyMultiPartitionTableMetaClient =
- StreamerUtil.createMetaClient(hoodieCatalog.inferTablePath(singleKeyMultiPartitionPath, singleKeyMultiPartitionTable), createHiveConf());
+ HoodieTableMetaClient singleKeyMultiPartitionTableMetaClient = HoodieTestUtils.createMetaClient(
+ createHiveConf(),
+ hoodieCatalog.inferTablePath(singleKeyMultiPartitionPath, singleKeyMultiPartitionTable));
assertThat(singleKeyMultiPartitionTableMetaClient.getTableConfig().getKeyGeneratorClassName(), is(ComplexAvroKeyGenerator.class.getName()));
// validate multiple key and single partition for partitioned table
@@ -244,8 +246,9 @@ public class TestHoodieHiveCatalog {
new CatalogTableImpl(multiKeySinglePartitionTableSchema, partitions, options, "hudi table");
hoodieCatalog.createTable(multiKeySinglePartitionPath, multiKeySinglePartitionTable, false);
- HoodieTableMetaClient multiKeySinglePartitionTableMetaClient =
- StreamerUtil.createMetaClient(hoodieCatalog.inferTablePath(multiKeySinglePartitionPath, multiKeySinglePartitionTable), createHiveConf());
+ HoodieTableMetaClient multiKeySinglePartitionTableMetaClient = HoodieTestUtils.createMetaClient(
+ createHiveConf(),
+ hoodieCatalog.inferTablePath(multiKeySinglePartitionPath, multiKeySinglePartitionTable));
assertThat(multiKeySinglePartitionTableMetaClient.getTableConfig().getKeyGeneratorClassName(), is(ComplexAvroKeyGenerator.class.getName()));
// validate key generator for non partitioned table
@@ -254,7 +257,8 @@ public class TestHoodieHiveCatalog {
new CatalogTableImpl(schema, new ArrayList<>(), options, "hudi table");
hoodieCatalog.createTable(nonPartitionPath, nonPartitionTable, false);
- metaClient = StreamerUtil.createMetaClient(hoodieCatalog.inferTablePath(nonPartitionPath, nonPartitionTable), createHiveConf());
+ metaClient = HoodieTestUtils.createMetaClient(
+ createHiveConf(), hoodieCatalog.inferTablePath(nonPartitionPath, nonPartitionTable));
keyGeneratorClassName = metaClient.getTableConfig().getKeyGeneratorClassName();
assertEquals(keyGeneratorClassName, NonpartitionedAvroKeyGenerator.class.getName());
}
@@ -322,7 +326,8 @@ public class TestHoodieHiveCatalog {
new CatalogTableImpl(schema, partitions, options, "hudi table");
hoodieCatalog.createTable(tablePath, table, true);
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(hoodieCatalog.inferTablePath(tablePath, table), createHiveConf());
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ createHiveConf(), hoodieCatalog.inferTablePath(tablePath, table));
return metaClient.getTableConfig().getProps();
}
@@ -449,7 +454,7 @@ public class TestHoodieHiveCatalog {
hoodieCatalog.dropPartition(tablePath, partitionSpec, false);
String tablePathStr = hoodieCatalog.inferTablePath(tablePath, hoodieCatalog.getTable(tablePath));
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tablePathStr, hoodieCatalog.getHiveConf());
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(hoodieCatalog.getHiveConf(), tablePathStr);
HoodieInstant latestInstant = metaClient.getActiveTimeline().filterCompletedInstants().lastInstant().orElse(null);
assertNotNull(latestInstant, "Delete partition commit should be completed");
HoodieCommitMetadata commitMetadata = WriteProfiles.getCommitMetadata(tablePath.getObjectName(), new org.apache.flink.core.fs.Path(tablePathStr),
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
index f69477c3df0..1999791ab30 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/table/format/TestInputFormat.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.source.IncrementalInputSplits;
@@ -776,7 +777,8 @@ public class TestInputFormat {
TestData.writeData(dataset, conf);
}
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tempFile.getAbsolutePath(), HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(conf), tempFile.getAbsolutePath());
List<String> commits = metaClient.getCommitsTimeline().filterCompletedInstants().getInstantsAsStream()
.map(HoodieInstant::getTimestamp).collect(Collectors.toList());
@@ -860,7 +862,8 @@ public class TestInputFormat {
TestData.writeDataAsBatch(dataset, conf);
}
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tempFile.getAbsolutePath(), HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(conf), tempFile.getAbsolutePath());
List<String> commits = metaClient.getCommitsTimeline().filterCompletedInstants().getInstantsAsStream()
.map(HoodieInstant::getTimestamp).collect(Collectors.toList());
@@ -1009,7 +1012,8 @@ public class TestInputFormat {
HoodieFlinkEngineContext.DEFAULT, FlinkWriteClients.getHoodieClientConfig(conf));
writeClient.clean();
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(tempFile.getAbsolutePath(), HadoopConfigurations.getHadoopConf(conf));
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(conf), tempFile.getAbsolutePath());
List<String> commits = metaClient.getCommitsTimeline().filterCompletedInstants().getInstantsAsStream()
.map(HoodieInstant::getTimestamp).collect(Collectors.toList());
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
index 42320bf55d5..b582c6293a9 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestData.java
@@ -40,7 +40,6 @@ import org.apache.hudi.sink.utils.StreamWriteFunctionWrapper;
import org.apache.hudi.sink.utils.TestFunctionWrapper;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.table.HoodieFlinkTable;
-import org.apache.hudi.util.StreamerUtil;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericRecord;
@@ -88,6 +87,7 @@ import java.util.stream.IntStream;
import static junit.framework.TestCase.assertEquals;
import static org.apache.hudi.common.table.HoodieTableConfig.HOODIE_PROPERTIES_FILE;
import static org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient;
import static org.apache.hudi.hadoop.utils.HoodieInputFormatUtils.HOODIE_RECORD_KEY_COL_POS;
import static org.apache.hudi.table.format.FormatUtils.buildAvroRecordBySchema;
import static org.hamcrest.CoreMatchers.is;
@@ -808,7 +808,7 @@ public class TestData {
Function<GenericRecord, String> extractor) throws IOException {
// 1. init flink table
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(basePath.toURI().toString(), new org.apache.hadoop.conf.Configuration());
+ HoodieTableMetaClient metaClient = createMetaClient(basePath.toURI().toString());
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath.toURI().toString()).build();
HoodieFlinkTable<?> table = HoodieFlinkTable.create(config, HoodieFlinkEngineContext.DEFAULT, metaClient);
@@ -864,7 +864,7 @@ public class TestData {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
.fromFile(hoodiePropertiesFile)
.withPath(basePath).build();
- HoodieTableMetaClient metaClient = StreamerUtil.createMetaClient(basePath, new org.apache.hadoop.conf.Configuration());
+ HoodieTableMetaClient metaClient = createMetaClient(basePath);
HoodieFlinkTable<?> table = HoodieFlinkTable.create(config, HoodieFlinkEngineContext.DEFAULT, metaClient);
Schema schema = new TableSchemaResolver(metaClient).getTableAvroSchema();
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestStreamerUtil.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestStreamerUtil.java
index 072e43bba7d..99ea23b7bca 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestStreamerUtil.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestStreamerUtil.java
@@ -20,6 +20,7 @@ package org.apache.hudi.utils;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
@@ -62,10 +63,7 @@ public class TestStreamerUtil {
StreamerUtil.initTableIfNotExists(conf);
// Validate the partition fields & preCombineField in hoodie.properties.
- HoodieTableMetaClient metaClient1 = HoodieTableMetaClient.builder()
- .setBasePath(tempFile.getAbsolutePath())
- .setConf(new org.apache.hadoop.conf.Configuration())
- .build();
+ HoodieTableMetaClient metaClient1 = HoodieTestUtils.createMetaClient(tempFile.getAbsolutePath());
assertTrue(metaClient1.getTableConfig().getPartitionFields().isPresent(),
"Missing partition columns in the hoodie.properties.");
assertArrayEquals(metaClient1.getTableConfig().getPartitionFields().get(), new String[] {"p0", "p1"});
@@ -76,10 +74,7 @@ public class TestStreamerUtil {
conf.removeConfig(FlinkOptions.PARTITION_PATH_FIELD);
FileIOUtils.deleteDirectory(tempFile);
StreamerUtil.initTableIfNotExists(conf);
- HoodieTableMetaClient metaClient2 = HoodieTableMetaClient.builder()
- .setBasePath(tempFile.getAbsolutePath())
- .setConf(new org.apache.hadoop.conf.Configuration())
- .build();
+ HoodieTableMetaClient metaClient2 = HoodieTestUtils.createMetaClient(tempFile.getAbsolutePath());
assertFalse(metaClient2.getTableConfig().getPartitionFields().isPresent());
assertEquals(metaClient2.getTableConfig().getKeyGeneratorClassName(), SimpleAvroKeyGenerator.class.getName());
}
diff --git a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java
index 6cb53c2b2d5..0ccf9f9b75a 100644
--- a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java
+++ b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/TestUtils.java
@@ -26,6 +26,7 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline;
import org.apache.hudi.common.table.timeline.TimelineUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.configuration.FlinkOptions;
import org.apache.hudi.configuration.HadoopConfigurations;
import org.apache.hudi.source.StreamReadMonitoringFunction;
@@ -46,20 +47,20 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
*/
public class TestUtils {
public static String getLastPendingInstant(String basePath) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return StreamerUtil.getLastPendingInstant(metaClient);
}
public static String getLastCompleteInstant(String basePath) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return StreamerUtil.getLastCompletedInstant(metaClient);
}
public static String getLastCompleteInstant(String basePath, String commitAction) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getCommitsTimeline().filterCompletedInstants()
.filter(instant -> commitAction.equals(instant.getAction()))
.lastInstant()
@@ -68,8 +69,8 @@ public class TestUtils {
}
public static String getLastDeltaCompleteInstant(String basePath) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getCommitsTimeline().filterCompletedInstants()
.filter(hoodieInstant -> hoodieInstant.getAction().equals(HoodieTimeline.DELTA_COMMIT_ACTION))
.lastInstant()
@@ -78,16 +79,16 @@ public class TestUtils {
}
public static String getFirstCompleteInstant(String basePath) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getCommitsAndCompactionTimeline().filterCompletedInstants().firstInstant()
.map(HoodieInstant::getTimestamp).orElse(null);
}
@Nullable
public static String getNthCompleteInstant(String basePath, int n, String action) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getActiveTimeline()
.filterCompletedInstants()
.filter(instant -> action.equals(instant.getAction()))
@@ -97,8 +98,8 @@ public class TestUtils {
@Nullable
public static String getNthArchivedInstant(String basePath, int n) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getArchivedTimeline().getCommitsTimeline().filterCompletedInstants()
.nthInstant(n).map(HoodieInstant::getTimestamp).orElse(null);
}
@@ -116,8 +117,8 @@ public class TestUtils {
}
public static int getCompletedInstantCount(String basePath, String action) {
- final HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(HadoopConfigurations.getHadoopConf(new Configuration())).setBasePath(basePath).build();
+ final HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ HadoopConfigurations.getHadoopConf(new Configuration()), basePath);
return metaClient.getActiveTimeline()
.filterCompletedInstants()
.filter(instant -> action.equals(instant.getAction()))
diff --git a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
index 0d10e602e4d..68201e43df3 100644
--- a/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
+++ b/hudi-integ-test/src/test/java/org/apache/hudi/integ/testsuite/job/TestHoodieTestSuiteJob.java
@@ -38,7 +38,6 @@ import org.apache.hudi.keygen.TimestampBasedKeyGenerator;
import org.apache.hudi.utilities.sources.AvroDFSSource;
import org.apache.hudi.utilities.testutils.UtilitiesTestBase;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.AfterEach;
@@ -53,6 +52,7 @@ import java.io.IOException;
import java.util.UUID;
import java.util.stream.Stream;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_URL;
import static org.apache.hudi.hive.testutils.HiveTestService.HS2_JDBC_URL;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
@@ -210,7 +210,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
cfg.workloadDagGenerator = ComplexDagGenerator.class.getName();
HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
hoodieTestSuiteJob.runTestSuite();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.targetBasePath);
assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().countInstants(), 2);
}
@@ -229,7 +229,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
}
HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
hoodieTestSuiteJob.runTestSuite();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.targetBasePath);
assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().countInstants(), 1);
}
@@ -244,7 +244,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
cfg.workloadYamlPath = basePath + "/" + COW_DAG_FILE_NAME;
HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
hoodieTestSuiteJob.runTestSuite();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.targetBasePath);
//assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().countInstants(), 5);
}
@@ -259,7 +259,7 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
cfg.workloadYamlPath = basePath + "/" + MOR_DAG_FILE_NAME;
HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
hoodieTestSuiteJob.runTestSuite();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.targetBasePath);
//assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().countInstants(), 7);
}
@@ -280,7 +280,8 @@ public class TestHoodieTestSuiteJob extends UtilitiesTestBase {
cfg.workloadYamlPath = basePath + "/" + COW_DAG_FILE_NAME_SPARK_DATASOURCE_NODES;
HoodieTestSuiteJob hoodieTestSuiteJob = new HoodieTestSuiteJob(cfg, jsc);
hoodieTestSuiteJob.runTestSuite();
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(new Configuration()).setBasePath(cfg.targetBasePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(cfg.targetBasePath);
+
assertEquals(metaClient.getActiveTimeline().getCommitsTimeline().countInstants(), 3);
}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/BaseProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/BaseProcedure.scala
index b06aea2ac58..3b4fe9ac0bd 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/BaseProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/BaseProcedure.scala
@@ -18,9 +18,11 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
+import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.config.{HoodieIndexConfig, HoodieWriteConfig}
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.index.HoodieIndex.IndexType
+
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.catalyst.InternalRow
@@ -39,6 +41,10 @@ abstract class BaseProcedure extends Procedure {
.build
}
+ protected def createMetaClient(jsc: JavaSparkContext, basePath: String): HoodieTableMetaClient = {
+ HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ }
+
protected def getParamKey(parameter: ProcedureParameter, isNamedArgs: Boolean): String = {
if (isNamedArgs) {
parameter.name
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala
index fdac678b477..d51c58289f5 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CommitsCompareProcedure.scala
@@ -18,11 +18,9 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieTimeline
+
import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util.function.Supplier
@@ -50,8 +48,8 @@ class CommitsCompareProcedure() extends BaseProcedure with ProcedureBuilder {
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val source = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
- val target = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(path).build
+ val source = createMetaClient(jsc, basePath)
+ val target = createMetaClient(jsc, path)
val sourceTimeline = source.getActiveTimeline.getCommitsTimeline.filterCompletedInstants
val targetTimeline = target.getActiveTimeline.getCommitsTimeline.filterCompletedInstants
val targetLatestCommit =
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateMetadataTableProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateMetadataTableProcedure.scala
index 7989a2d6cd2..acadd92776f 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateMetadataTableProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateMetadataTableProcedure.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.metadata.{HoodieTableMetadata, SparkHoodieBackedTableMetadataWriter}
import org.apache.hudi.storage.StoragePath
@@ -49,7 +48,7 @@ class CreateMetadataTableProcedure extends BaseProcedure with ProcedureBuilder w
val tableName = getArgValueOrDefault(args, PARAMETERS(0))
val basePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val metadataPath = new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(basePath))
try {
@@ -65,7 +64,7 @@ class CreateMetadataTableProcedure extends BaseProcedure with ProcedureBuilder w
val timer = HoodieTimer.start
val writeConfig = getWriteConfig(basePath)
SparkHoodieBackedTableMetadataWriter.create(metaClient.getHadoopConf, writeConfig, new HoodieSparkEngineContext(jsc))
- Seq(Row("Created Metadata Table in " + metadataPath + " (duration=" + timer.endTimer / 1000.0 + "secs)"))
+ Seq(Row("Created Metadata Table in " + metadataPath + " (duration=" + timer.endTimer / 1000.0 + "secs)"))
}
override def build = new CreateMetadataTableProcedure()
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateSavepointProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateSavepointProcedure.scala
index ebaa262d853..0ae22f54af7 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateSavepointProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/CreateSavepointProcedure.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieTimeline
import org.apache.hudi.common.util.StringUtils
import org.apache.hudi.exception.{HoodieException, HoodieSavepointException}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -55,7 +55,7 @@ class CreateSavepointProcedure extends BaseProcedure with ProcedureBuilder with
val comments = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[String]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val completedTimeline: HoodieTimeline = metaClient.getCommitsTimeline.filterCompletedInstants
if (StringUtils.isNullOrEmpty(commitTime)) {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMetadataTableProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMetadataTableProcedure.scala
index 540151bf67d..69057056292 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMetadataTableProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteMetadataTableProcedure.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.metadata.HoodieTableMetadataUtil.deleteMetadataTable
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
@@ -48,7 +48,7 @@ class DeleteMetadataTableProcedure extends BaseProcedure with ProcedureBuilder w
var metadataPaths = ""
for (tb <- tableNames) {
val basePath = getBasePath(Option.apply(tb))
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
try {
val metadataTableBasePath = deleteMetadataTable(metaClient, new HoodieSparkEngineContext(jsc), false)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteSavepointProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteSavepointProcedure.scala
index d568566e554..d9a6dc4197d 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteSavepointProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/DeleteSavepointProcedure.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.util.StringUtils
import org.apache.hudi.exception.{HoodieException, HoodieSavepointException}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -51,7 +51,7 @@ class DeleteSavepointProcedure extends BaseProcedure with ProcedureBuilder with
var instantTime = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[String]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val completedInstants = metaClient.getActiveTimeline.getSavePointTimeline.filterCompletedInstants
if (completedInstants.empty) throw new HoodieException("There are no completed savepoint to run delete")
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala
index dbe390b81ce..0745b14aec3 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ExportInstantsProcedure.scala
@@ -28,7 +28,7 @@ import org.apache.hudi.common.table.log.block.HoodieAvroDataBlock
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline, TimelineMetadataUtils}
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.hadoop.fs.HadoopFSUtils
-import org.apache.hudi.storage.{StoragePath, HoodieStorage, HoodieStorageUtils}
+import org.apache.hudi.storage.{HoodieStorage, HoodieStorageUtils, StoragePath}
import org.apache.avro.generic.GenericRecord
import org.apache.avro.specific.SpecificData
@@ -40,7 +40,6 @@ import java.io.File
import java.util
import java.util.Collections
import java.util.function.Supplier
-
import scala.collection.JavaConverters._
import scala.util.control.Breaks.break
@@ -76,7 +75,7 @@ class ExportInstantsProcedure extends BaseProcedure with ProcedureBuilder with L
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val archivePath = new Path(basePath + "/.hoodie/.commits_.archive*")
val actionSet: util.Set[String] = Set(actions.split(","): _*).asJava
val numExports = if (limit == -1) Integer.MAX_VALUE else limit
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/InitMetadataTableProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/InitMetadataTableProcedure.scala
index 7d90ce57944..58a84d0c74d 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/InitMetadataTableProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/InitMetadataTableProcedure.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.client.common.HoodieSparkEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.metadata.{HoodieTableMetadata, SparkHoodieBackedTableMetadataWriter}
import org.apache.hudi.storage.StoragePath
@@ -52,7 +51,7 @@ class InitMetadataTableProcedure extends BaseProcedure with ProcedureBuilder wit
val readOnly = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Boolean]
val basePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val metadataPath = new StoragePath(HoodieTableMetadata.getMetadataTableBasePath(basePath))
try {
metaClient.getStorage.listDirectEntries(metadataPath)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
index 3ae183101e8..eff7df01fb8 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairAddpartitionmetaProcedure.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodiePartitionMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.storage.StoragePath
import org.apache.spark.internal.Logging
@@ -28,7 +27,6 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.Supplier
-
import scala.collection.JavaConversions._
class RepairAddpartitionmetaProcedure extends BaseProcedure with ProcedureBuilder with Logging {
@@ -54,7 +52,7 @@ class RepairAddpartitionmetaProcedure extends BaseProcedure with ProcedureBuilde
val dryRun = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Boolean]
val tablePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(tablePath).build
+ val metaClient = createMetaClient(jsc, tablePath)
val latestCommit: String = metaClient.getActiveTimeline.getCommitTimeline.lastInstant.get.getTimestamp
val partitionPaths: util.List[String] = FSUtils.getAllPartitionFoldersThreeLevelsDown(metaClient.getStorage, tablePath);
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala
index 28d2fbf940a..e0e0db63a83 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairCorruptedCleanFilesProcedure.scala
@@ -17,11 +17,11 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.avro.AvroRuntimeException
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstant}
import org.apache.hudi.common.util.CleanerUtils
import org.apache.hudi.exception.HoodieIOException
+
+import org.apache.avro.AvroRuntimeException
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -49,7 +49,7 @@ class RepairCorruptedCleanFilesProcedure extends BaseProcedure with ProcedureBui
val tableName = getArgValueOrDefault(args, PARAMETERS(0))
val tablePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(tablePath).build
+ val metaClient = createMetaClient(jsc, tablePath)
val cleanerTimeline = metaClient.getActiveTimeline.getCleanerTimeline
logInfo("Inspecting pending clean metadata in timeline for corrupted files")
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
index 54019b0bc76..e9d76ef2631 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RepairOverwriteHoodiePropsProcedure.scala
@@ -17,23 +17,20 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.common.table.HoodieTableMetaClient.METAFOLDER_NAME
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.storage.StoragePath
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
-import java.io.FileInputStream
import java.util
import java.util.Properties
import java.util.function.Supplier
-
import scala.collection.JavaConversions._
import scala.collection.JavaConverters.asScalaIteratorConverter
@@ -68,7 +65,7 @@ class RepairOverwriteHoodiePropsProcedure extends BaseProcedure with ProcedureBu
val overwriteFilePath = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[String]
val tablePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(tablePath).build
+ val metaClient = createMetaClient(jsc, tablePath)
var newProps = new Properties
loadNewProps(overwriteFilePath, newProps)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToSavepointProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToSavepointProcedure.scala
index f0c138d1062..80688838bd2 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToSavepointProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RollbackToSavepointProcedure.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.util.StringUtils
import org.apache.hudi.exception.{HoodieException, HoodieSavepointException}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -51,7 +51,7 @@ class RollbackToSavepointProcedure extends BaseProcedure with ProcedureBuilder w
var instantTime = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[String]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val completedInstants = metaClient.getActiveTimeline.getSavePointTimeline.filterCompletedInstants
if (completedInstants.empty) throw new HoodieException("There are no completed savepoint to run delete")
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
index 27f92027a02..51468dec8e2 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunClusteringProcedure.scala
@@ -26,6 +26,7 @@ import org.apache.hudi.common.util.{ClusteringUtils, HoodieTimer, Option => HOpt
import org.apache.hudi.config.{HoodieClusteringConfig, HoodieLockConfig}
import org.apache.hudi.exception.HoodieClusteringException
import org.apache.hudi.{AvroConversionUtils, HoodieCLIUtils, HoodieFileIndex}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.HoodieCatalystExpressionUtils.{resolveExpr, splitPartitionAndDataPredicates}
import org.apache.spark.sql.Row
@@ -34,6 +35,7 @@ import org.apache.spark.sql.execution.datasources.FileStatusCache
import org.apache.spark.sql.types._
import java.util.function.Supplier
+
import scala.collection.JavaConverters._
class RunClusteringProcedure extends BaseProcedure
@@ -85,7 +87,7 @@ class RunClusteringProcedure extends BaseProcedure
val parts = getArgValueOrDefault(args, PARAMETERS(9))
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
var confs: Map[String, String] = Map.empty
val selectedPartitions: String = (parts, predicate) match {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala
index 68a28b5fd54..f17acf20fec 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/RunCompactionProcedure.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.client.SparkRDDWriteClient
import org.apache.hudi.common.model.HoodieCommitMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieTimeline}
import org.apache.hudi.common.util.{CompactionUtils, HoodieTimer, Option => HOption}
import org.apache.hudi.config.HoodieLockConfig
@@ -31,6 +30,7 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import java.util.function.Supplier
+
import scala.collection.JavaConversions._
import scala.collection.JavaConverters._
@@ -82,7 +82,7 @@ class RunCompactionProcedure extends BaseProcedure with ProcedureBuilder with Sp
}
val basePath = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
if (metaClient.getTableConfig.isMetadataTableAvailable) {
if (!confs.contains(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key)) {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala
index a63125374dd..fb6394ea84c 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowArchivedCommitsProcedure.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.HoodieCommitMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieDefaultTimeline, HoodieInstant}
import org.apache.hudi.common.util.StringUtils
+
import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.time.ZonedDateTime
@@ -87,7 +85,7 @@ class ShowArchivedCommitsProcedure(includeExtraMetadata: Boolean) extends BasePr
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
// start time for commits, default: now - 10 days
// end time for commits, default: now - 1 day
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala
index 958f37c5881..08add1b0793 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapMappingProcedure.scala
@@ -21,6 +21,7 @@ import org.apache.hudi.common.bootstrap.index.BootstrapIndex
import org.apache.hudi.common.model.{BootstrapFileMapping, HoodieFileGroupId}
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -62,7 +63,7 @@ class ShowBootstrapMappingProcedure extends BaseProcedure with ProcedureBuilder
val desc = getArgValueOrDefault(args, PARAMETERS(5)).get.asInstanceOf[Boolean]
val basePath: String = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
if (partitionPath.isEmpty && fileIds.nonEmpty) throw new IllegalStateException("PartitionPath is mandatory when passing fileIds.")
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapPartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapPartitionsProcedure.scala
index c62bcfa73e9..71486d7b8d0 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapPartitionsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowBootstrapPartitionsProcedure.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.bootstrap.index.BootstrapIndex
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -44,7 +45,7 @@ class ShowBootstrapPartitionsProcedure extends BaseProcedure with ProcedureBuild
val tableName = getArgValueOrDefault(args, PARAMETERS(0))
val basePath: String = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val indexReader = createBootstrapIndexReader(metaClient)
val indexedPartitions = indexReader.getIndexedPartitionPaths
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala
index 69aae49466e..d37a4720ac6 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowClusteringProcedure.scala
@@ -17,16 +17,15 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.hudi.{HoodieCLIUtils, SparkAdapterSupport}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieTimeline
import org.apache.hudi.common.util.ClusteringUtils
+import org.apache.hudi.{HoodieCLIUtils, SparkAdapterSupport}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import java.util.function.Supplier
-
import scala.collection.JavaConverters._
class ShowClusteringProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging {
@@ -57,7 +56,7 @@ class ShowClusteringProcedure extends BaseProcedure with ProcedureBuilder with S
val showInvolvedPartitions = getArgValueOrDefault(args, PARAMETERS(3)).get.asInstanceOf[Boolean]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val clusteringInstants = metaClient.getActiveTimeline.getInstants.iterator().asScala
.filter(p => p.getAction == HoodieTimeline.REPLACE_COMMIT_ACTION)
.toSeq
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitExtraMetadataProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitExtraMetadataProcedure.scala
index e80fc2b36db..393fc31abb3 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitExtraMetadataProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitExtraMetadataProcedure.scala
@@ -19,9 +19,9 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieReplaceCommitMetadata}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -58,7 +58,7 @@ class ShowCommitExtraMetadataProcedure() extends BaseProcedure with ProcedureBui
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
val timeline = activeTimeline.getCommitsTimeline.filterCompletedInstants
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala
index 407ebcf76d1..fce0dfab82f 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitFilesProcedure.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieReplaceCommitMetadata, HoodieWriteStat}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
@@ -64,7 +62,7 @@ class ShowCommitFilesProcedure() extends BaseProcedure with ProcedureBuilder {
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
val timeline = activeTimeline.getCommitsTimeline.filterCompletedInstants
val hoodieInstantOption = getCommitForInstant(timeline, instantTime)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala
index 8439ebf9374..9a65c0d24ab 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitPartitionsProcedure.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieReplaceCommitMetadata, HoodieWriteStat}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
@@ -63,7 +61,7 @@ class ShowCommitPartitionsProcedure() extends BaseProcedure with ProcedureBuilde
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
val timeline = activeTimeline.getCommitsTimeline.filterCompletedInstants
val hoodieInstantOption = getCommitForInstant(timeline, instantTime)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala
index 50d55d47557..651e4e52d3c 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitWriteStatsProcedure.scala
@@ -19,12 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.{HoodieCommitMetadata, HoodieReplaceCommitMetadata}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.TableIdentifier
-import org.apache.spark.sql.catalyst.catalog.HoodieCatalogTable
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
@@ -58,7 +56,7 @@ class ShowCommitWriteStatsProcedure() extends BaseProcedure with ProcedureBuilde
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
val timeline = activeTimeline.getCommitsTimeline.filterCompletedInstants
val hoodieInstantOption = getCommitForInstant(timeline, instantTime)
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala
index 8f8ebd9ce29..7b4af9d37af 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCommitsProcedure.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.HoodieCLIUtils
import org.apache.hudi.common.model.HoodieCommitMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieDefaultTimeline, HoodieInstant}
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -82,7 +82,7 @@ class ShowCommitsProcedure(includeExtraMetadata: Boolean) extends BaseProcedure
val hoodieCatalogTable = HoodieCLIUtils.getHoodieCatalogTable(sparkSession, table)
val basePath = hoodieCatalogTable.tableLocation
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
if (includeExtraMetadata) {
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala
index 5aee4bf3a12..6a0a8d1a1ae 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowCompactionProcedure.scala
@@ -19,7 +19,6 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.SparkAdapterSupport
import org.apache.hudi.common.model.HoodieTableType
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieTimeline
import org.apache.hudi.common.util.CompactionUtils
@@ -28,7 +27,6 @@ import org.apache.spark.sql.Row
import org.apache.spark.sql.types._
import java.util.function.Supplier
-
import scala.collection.JavaConverters._
class ShowCompactionProcedure extends BaseProcedure with ProcedureBuilder with SparkAdapterSupport with Logging {
@@ -60,7 +58,7 @@ class ShowCompactionProcedure extends BaseProcedure with ProcedureBuilder with S
val limit = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Int]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
assert(metaClient.getTableType == HoodieTableType.MERGE_ON_READ,
s"Cannot show compaction on a Non Merge On Read table.")
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala
index 9388cb286ba..3271aed96b0 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowFileSystemViewProcedure.scala
@@ -91,7 +91,7 @@ class ShowFileSystemViewProcedure(showLatest: Boolean) extends BaseProcedure wit
excludeCompaction: Boolean
): HoodieTableFileSystemView = {
val basePath = getBasePath(table)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val storage = metaClient.getStorage
val statuses = if (globRegex == PARAMETERS_ALL.apply(6).default) {
FSUtils.getAllDataPathInfo(storage, new StoragePath(basePath))
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala
index 5941af9b0c8..36f4ad4b1bc 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileMetadataProcedure.scala
@@ -20,11 +20,11 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodieLogFile
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
+import org.apache.hudi.common.table.TableSchemaResolver
import org.apache.hudi.common.table.log.HoodieLogFormat
-import org.apache.hudi.common.table.log.block.{HoodieCorruptBlock, HoodieDataBlock}
import org.apache.hudi.common.table.log.block.HoodieLogBlock.{HeaderMetadataType, HoodieLogBlockType}
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.storage.{StoragePath, HoodieStorageUtils}
+import org.apache.hudi.common.table.log.block.{HoodieCorruptBlock, HoodieDataBlock}
+import org.apache.hudi.storage.StoragePath
import com.fasterxml.jackson.databind.ObjectMapper
import org.apache.parquet.avro.AvroSchemaConverter
@@ -34,7 +34,6 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util.Objects
import java.util.concurrent.atomic.AtomicInteger
import java.util.function.Supplier
-
import scala.collection.JavaConverters.{asScalaBufferConverter, asScalaIteratorConverter, mapAsScalaMapConverter}
class ShowHoodieLogFileMetadataProcedure extends BaseProcedure with ProcedureBuilder {
@@ -58,7 +57,7 @@ class ShowHoodieLogFileMetadataProcedure extends BaseProcedure with ProcedureBui
val logFilePathPattern: String = getArgValueOrDefault(args, parameters(1)).get.asInstanceOf[String]
val limit: Int = getArgValueOrDefault(args, parameters(2)).get.asInstanceOf[Int]
val basePath = getBasePath(table)
- val storage = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build.getStorage
+ val storage = createMetaClient(jsc, basePath).getStorage
val logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(storage, new StoragePath(logFilePathPattern)).iterator().asScala
.map(_.getPath.toString).toList
val commitCountAndMetadata =
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
index c751682968f..97137c5ae51 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowHoodieLogFileRecordsProcedure.scala
@@ -21,9 +21,9 @@ import org.apache.hudi.common.config.HoodieCommonConfig
import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
import org.apache.hudi.common.model.{HoodieLogFile, HoodieRecordPayload}
+import org.apache.hudi.common.table.TableSchemaResolver
import org.apache.hudi.common.table.log.block.HoodieDataBlock
import org.apache.hudi.common.table.log.{HoodieLogFormat, HoodieMergedLogRecordScanner}
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.common.util.{FileIOUtils, ValidationUtils}
import org.apache.hudi.config.{HoodieCompactionConfig, HoodieMemoryConfig}
import org.apache.hudi.storage.StoragePath
@@ -57,7 +57,7 @@ class ShowHoodieLogFileRecordsProcedure extends BaseProcedure with ProcedureBuil
val merge: Boolean = getArgValueOrDefault(args, parameters(2)).get.asInstanceOf[Boolean]
val limit: Int = getArgValueOrDefault(args, parameters(3)).get.asInstanceOf[Int]
val basePath = getBasePath(table)
- val client = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val client = createMetaClient(jsc, basePath)
val storage = client.getStorage
val logFilePaths = FSUtils.getGlobStatusExcludingMetaFolder(storage, new StoragePath(logFilePathPattern)).iterator().asScala
.map(_.getPath.toString).toList
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
index 2d7704420be..e17c8e12dca 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableFilesProcedure.scala
@@ -19,11 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.{HoodieTimer, StringUtils}
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.metadata.HoodieBackedTableMetadata
-import org.apache.hudi.storage.{StoragePathInfo, StoragePath}
+import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -31,7 +30,6 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.function.Supplier
-
import scala.jdk.CollectionConverters.asScalaBufferConverter
class ShowMetadataTableFilesProcedure() extends BaseProcedure with ProcedureBuilder with Logging {
@@ -55,7 +53,7 @@ class ShowMetadataTableFilesProcedure() extends BaseProcedure with ProcedureBuil
val partition = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[String]
val basePath = getBasePath(table)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
val metaReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getHadoopConf), config, basePath)
if (!metaReader.enabled){
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
index 096a0ff1e3f..d517f5386d5 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowMetadataTableStatsProcedure.scala
@@ -19,8 +19,8 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.metadata.HoodieBackedTableMetadata
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -48,7 +48,7 @@ class ShowMetadataTableStatsProcedure() extends BaseProcedure with ProcedureBuil
val table = getArgValueOrDefault(args, PARAMETERS(0))
val basePath = getBasePath(table)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
val metadata = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getHadoopConf), config, basePath)
val stats = metadata.stats
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowRollbacksProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowRollbacksProcedure.scala
index 8516b8bef2c..edd47f5cad6 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowRollbacksProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowRollbacksProcedure.scala
@@ -17,19 +17,18 @@
package org.apache.spark.sql.hudi.command.procedures
-import java.io.IOException
-import java.util
-import java.util.function.Supplier
-
import org.apache.hudi.avro.model.HoodieRollbackMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieInstant.State
import org.apache.hudi.common.table.timeline.HoodieTimeline.ROLLBACK_ACTION
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstant, TimelineMetadataUtils}
import org.apache.hudi.exception.HoodieException
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
+import java.io.IOException
+import java.util
+import java.util.function.Supplier
import scala.collection.JavaConversions.asScalaBuffer
import scala.collection.JavaConverters._
@@ -72,7 +71,7 @@ class ShowRollbacksProcedure(showDetails: Boolean) extends BaseProcedure with Pr
val limit = getArgValueOrDefault(args, parameters(1)).get.asInstanceOf[Int]
val basePath = getBasePath(tableName)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline = metaClient.getActiveTimeline
if (showDetails) {
val instantTime = getArgValueOrDefault(args, parameters(2)).get.asInstanceOf[String]
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowSavepointsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowSavepointsProcedure.scala
index 3a789f95105..15c80893369 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowSavepointsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowSavepointsProcedure.scala
@@ -17,8 +17,8 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstant, HoodieTimeline}
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -47,7 +47,7 @@ class ShowSavepointsProcedure extends BaseProcedure with ProcedureBuilder {
val tablePath = getArgValueOrDefault(args, PARAMETERS(1))
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val activeTimeline: HoodieActiveTimeline = metaClient.getActiveTimeline
val timeline: HoodieTimeline = activeTimeline.getSavePointTimeline.filterCompletedInstants
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala
index 9846a2906e1..f08da9483bd 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowTablePropertiesProcedure.scala
@@ -17,7 +17,6 @@
package org.apache.spark.sql.hudi.command.procedures
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -49,7 +48,7 @@ class ShowTablePropertiesProcedure() extends BaseProcedure with ProcedureBuilder
val limit = getArgValueOrDefault(args, PARAMETERS(2)).get.asInstanceOf[Int]
val basePath: String = getBasePath(tableName, tablePath)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val tableProps = metaClient.getTableConfig.getProps
val rows = new util.ArrayList[Row]
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala
index 6377a817b22..cb5c0d67b66 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsFileSizeProcedure.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.ValidationUtils
import org.apache.hudi.storage.StoragePath
@@ -28,7 +27,6 @@ import org.apache.spark.sql.hudi.command.procedures.StatsFileSizeProcedure.MAX_F
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util.function.Supplier
-
import scala.collection.JavaConverters.{asScalaBufferConverter, mapAsScalaMapConverter}
class StatsFileSizeProcedure extends BaseProcedure with ProcedureBuilder {
@@ -66,7 +64,7 @@ class StatsFileSizeProcedure extends BaseProcedure with ProcedureBuilder {
val globRegex = getArgValueOrDefault(args, parameters(1)).get.asInstanceOf[String]
val limit: Int = getArgValueOrDefault(args, parameters(2)).get.asInstanceOf[Int]
val basePath = getBasePath(table)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val storage = metaClient.getStorage
val isTablePartitioned = metaClient.getTableConfig.isTablePartitioned
val maximumPartitionDepth = if (isTablePartitioned) metaClient.getTableConfig.getPartitionFields.get.length else 0
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala
index 0c0f55cca5e..36be3b14678 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/StatsWriteAmplificationProcedure.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.model.HoodieCommitMetadata
-import org.apache.hudi.common.table.HoodieTableMetaClient
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -44,7 +44,7 @@ class StatsWriteAmplificationProcedure extends BaseProcedure with ProcedureBuild
val table = getArgValueOrDefault(args, parameters(0))
val limit: Int = getArgValueOrDefault(args, parameters(1)).get.asInstanceOf[Int]
val basePath = getBasePath(table)
- val client = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val client = createMetaClient(jsc, basePath)
val activeTimeline = client.getActiveTimeline
val timeline = activeTimeline.getCommitTimeline.filterCompletedInstants()
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala
index 774baf854a1..10a10160745 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateHoodieSyncProcedure.scala
@@ -21,6 +21,7 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.model.HoodieCommitMetadata
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
+
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
@@ -29,7 +30,6 @@ import org.joda.time.DateTime
import java.io.IOException
import java.sql.{Connection, DriverManager, ResultSet, SQLException}
import java.util.function.Supplier
-
import scala.collection.JavaConverters._
class ValidateHoodieSyncProcedure extends BaseProcedure with ProcedureBuilder with Logging {
@@ -79,8 +79,8 @@ class ValidateHoodieSyncProcedure extends BaseProcedure with ProcedureBuilder wi
val srcBasePath = getBasePath(srcTable, Option.empty)
val dstBasePath = getBasePath(dstTable, Option.empty)
- val srcMetaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(srcBasePath).build
- val targetMetaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(dstBasePath).build
+ val srcMetaClient = createMetaClient(jsc, srcBasePath)
+ val targetMetaClient = createMetaClient(jsc, dstBasePath)
val targetTimeline = targetMetaClient.getActiveTimeline.getCommitsTimeline
val sourceTimeline = srcMetaClient.getActiveTimeline.getCommitsTimeline
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
index 35ef5d4c545..18e7ed63c2d 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ValidateMetadataTableFilesProcedure.scala
@@ -19,11 +19,10 @@ package org.apache.spark.sql.hudi.command.procedures
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.engine.HoodieLocalEngineContext
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.HoodieTimer
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.metadata.HoodieBackedTableMetadata
-import org.apache.hudi.storage.{StoragePathInfo, StoragePath}
+import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
@@ -32,7 +31,6 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType}
import java.util
import java.util.Collections
import java.util.function.Supplier
-
import scala.collection.JavaConversions._
import scala.jdk.CollectionConverters.asScalaBufferConverter
@@ -62,12 +60,12 @@ class ValidateMetadataTableFilesProcedure() extends BaseProcedure with Procedure
val verbose = getArgValueOrDefault(args, PARAMETERS(1)).get.asInstanceOf[Boolean]
val basePath = getBasePath(table)
- val metaClient = HoodieTableMetaClient.builder.setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build
+ val metaClient = createMetaClient(jsc, basePath)
val config = HoodieMetadataConfig.newBuilder.enable(true).build
val metadataReader = new HoodieBackedTableMetadata(new HoodieLocalEngineContext(metaClient.getHadoopConf),
config, basePath)
- if (!metadataReader.enabled){
+ if (!metadataReader.enabled) {
throw new HoodieException(s"Metadata Table not enabled/initialized.")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java
index 8e5897d14e1..1a3b1d37247 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/HoodieJavaStreamingApp.java
@@ -30,6 +30,7 @@ import org.apache.hudi.exception.TableNotFoundException;
import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.MultiPartKeysValueExtractor;
import org.apache.hudi.hive.SlashEncodedDayPartitionValueExtractor;
+import org.apache.hudi.testutils.HoodieClientTestUtils;
import com.beust.jcommander.JCommander;
import com.beust.jcommander.Parameter;
@@ -196,7 +197,7 @@ public class HoodieJavaStreamingApp {
executor.shutdownNow();
}
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(jssc.hadoopConfiguration()).setBasePath(tablePath).build();
+ HoodieTableMetaClient metaClient = HoodieClientTestUtils.createMetaClient(jssc, tablePath);
if (tableType.equals(HoodieTableType.MERGE_ON_READ.name())) {
// Ensure we have successfully completed one compaction commit
ValidationUtils.checkArgument(metaClient.getActiveTimeline().getCommitTimeline().countInstants() == 1);
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestWriteClient.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestWriteClient.java
index 7acf6b2b6b0..e6363eac1a7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestWriteClient.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestWriteClient.java
@@ -71,7 +71,7 @@ public class TestWriteClient extends HoodieSparkClientTestBase {
result = client.insert(emptyRdd, secondCommit);
assertTrue(client.commit(secondCommit, result), "Commit should succeed");
// Schema Validations.
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder().setConf(jsc.hadoopConfiguration()).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = createMetaClient(jsc, basePath);
HoodieActiveTimeline timeline = metaClient.getActiveTimeline();
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromBytes(timeline.getInstantDetails(timeline.lastInstant().get()).get(), HoodieCommitMetadata.class);
assertTrue(metadata.getExtraMetadata().get("schema").isEmpty());
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
index 120304c1219..e3c3f0f6842 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkSqlWriter.scala
@@ -17,8 +17,6 @@
package org.apache.hudi
-import org.apache.avro.Schema
-import org.apache.commons.io.FileUtils
import org.apache.hudi.client.SparkRDDWriteClient
import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord, HoodieRecordPayload, HoodieTableType, WriteOperationType}
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver}
@@ -28,7 +26,11 @@ import org.apache.hudi.exception.{HoodieException, SchemaCompatibilityException}
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode
import org.apache.hudi.functional.TestBootstrap
import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator, SimpleKeyGenerator}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieClientTestUtils}
+
+import org.apache.avro.Schema
+import org.apache.commons.io.FileUtils
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.functions.{expr, lit}
@@ -1013,9 +1015,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
| )
| location '$tablePath1'
""".stripMargin)
- val tableConfig1 = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath1).build().getTableConfig
+ val tableConfig1 = createMetaClient(spark, tablePath1).getTableConfig
assert(tableConfig1.getHiveStylePartitioningEnable == "true")
assert(tableConfig1.getUrlEncodePartitioning == "false")
assert(tableConfig1.getKeyGeneratorClassName == classOf[SimpleKeyGenerator].getName)
@@ -1034,9 +1034,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
.option(HoodieWriteConfig.TBL_NAME.key, tableName2)
.option(DataSourceWriteOptions.URL_ENCODE_PARTITIONING.key, "true")
.mode(SaveMode.Overwrite).save(tablePath2)
- val tableConfig2 = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath2).build().getTableConfig
+ val tableConfig2 = createMetaClient(spark, tablePath2).getTableConfig
assert(tableConfig2.getHiveStylePartitioningEnable == "false")
assert(tableConfig2.getUrlEncodePartitioning == "true")
assert(tableConfig2.getKeyGeneratorClassName == classOf[SimpleKeyGenerator].getName)
@@ -1234,10 +1232,7 @@ def testBulkInsertForDropPartitionColumn(): Unit = {
}
private def fetchActualSchema(): Schema = {
- val tableMetaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tempBasePath)
- .build()
+ val tableMetaClient = createMetaClient(spark, tempBasePath)
new TableSchemaResolver(tableMetaClient).getTableAvroSchema(false)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestTableSchemaResolverWithSparkSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestTableSchemaResolverWithSparkSQL.scala
index 70886d96444..938c739c92e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestTableSchemaResolverWithSparkSQL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestTableSchemaResolverWithSparkSQL.scala
@@ -17,14 +17,16 @@
package org.apache.hudi
-import org.apache.avro.Schema
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.Path
import org.apache.hudi.avro.HoodieAvroUtils
import org.apache.hudi.avro.model.HoodieMetadataRecord
import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.testutils.DataSourceTestUtils
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
+import org.apache.avro.Schema
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.Path
import org.apache.spark.sql.SaveMode
import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
import org.junit.jupiter.api.{Tag, Test}
@@ -65,10 +67,7 @@ class TestTableSchemaResolverWithSparkSQL extends HoodieSparkWriterTestBase {
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Append, fooTableModifier, df1)
val metadataTablePath = tempPath.toAbsolutePath.toString + "/.hoodie/metadata"
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(metadataTablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, metadataTablePath)
// Delete latest metadata table deltacommit
// Get schema from metadata table hfile format base file.
@@ -107,10 +106,7 @@ class TestTableSchemaResolverWithSparkSQL extends HoodieSparkWriterTestBase {
val df1 = spark.createDataFrame(sc.parallelize(recordsSeq), structType)
HoodieSparkSqlWriter.write(sqlContext, SaveMode.Overwrite, fooTableModifier, df1)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tempPath.toAbsolutePath.toString)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tempPath.toAbsolutePath.toString)
assertTrue(new TableSchemaResolver(metaClient).hasOperationField)
schemaValuationBasedOnDataFile(metaClient, schema.toString())
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
index 1e55d5491b8..63225574b49 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestBasicSchemaEvolution.scala
@@ -17,11 +17,9 @@
package org.apache.hudi.functional
-import org.apache.hudi.{AvroConversionUtils, DataSourceWriteOptions, ScalaAssertionSupport}
import org.apache.hudi.HoodieConversionUtils.toJavaOption
import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType, OverwriteWithLatestAvroPayload}
-import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.common.util
+import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver}
import org.apache.hudi.common.util.Option
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.SchemaCompatibilityException
@@ -31,9 +29,6 @@ import org.apache.hudi.util.JFunction
import org.apache.hudi.{AvroConversionUtils, DataSourceWriteOptions, ScalaAssertionSupport}
import org.apache.hadoop.fs.FileSystem
-
-import org.apache.hadoop.fs.FileSystem
-import org.apache.spark.sql.{functions, HoodieUnsafeUtils, Row, SaveMode, SparkSession, SparkSessionExtensions}
import org.apache.spark.sql.hudi.HoodieSparkSessionExtension
import org.apache.spark.sql.types.{IntegerType, LongType, StringType, StructField, StructType}
import org.apache.spark.sql.{HoodieUnsafeUtils, Row, SaveMode, SparkSession, SparkSessionExtensions, functions}
@@ -43,7 +38,6 @@ import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.CsvSource
import java.util.function.Consumer
-
import scala.collection.JavaConversions.asScalaBuffer
import scala.collection.JavaConverters._
@@ -125,10 +119,7 @@ class TestBasicSchemaEvolution extends HoodieSparkClientTestBase with ScalaAsser
}
def loadTable(loadAllVersions: Boolean = true): (StructType, Seq[Row]) = {
- val tableMetaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(basePath)
- .build()
+ val tableMetaClient = createMetaClient(spark, basePath)
tableMetaClient.reloadActiveTimeline()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index f710786e41f..f886cc7ecef 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -31,8 +31,8 @@ import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType}
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline, TimelineUtils}
import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.{deleteRecordsToStrings, recordsToStrings}
+import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestUtils}
import org.apache.hudi.common.util.{ClusteringUtils, Option}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.config.metrics.HoodieMetricsConfig
@@ -50,6 +50,8 @@ import org.apache.hudi.testutils.HoodieSparkClientTestBase
import org.apache.hudi.util.JFunction
import org.apache.hudi.{AvroConversionUtils, DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers, QuickstartUtils, ScalaAssertionSupport}
+import org.apache.hadoop.fs.FileSystem
+import org.apache.spark.sql.functions.{col, concat, lit, udf, when}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.{FileSystem, Path, PathFilter}
import org.apache.spark.sql._
@@ -180,7 +182,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH + "'").count() > 0)
assertTrue(snapshot0.filter("_hoodie_partition_path = '" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH + "'").count() > 0)
- val storage = HoodieStorageUtils.getStorage(new StoragePath(basePath), new Configuration())
+ val storage = HoodieStorageUtils.getStorage(new StoragePath(basePath), HoodieTestUtils.getDefaultHadoopConf)
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH)))
assertTrue(storage.exists(new StoragePath(basePath + "/" + HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH)))
@@ -546,10 +548,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.options(options)
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val commit1Time = metaClient.getActiveTimeline.lastInstant().get().getTimestamp
val dataGen2 = new HoodieTestDataGenerator(Array("2022-01-02"))
@@ -612,7 +611,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.save(basePath)
}
- val tableMetaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath).build()
+ val tableMetaClient = createMetaClient(spark, basePath)
assertFalse(tableMetaClient.getArchivedTimeline.empty())
val actualSchema = new TableSchemaResolver(tableMetaClient).getTableAvroSchema(false)
@@ -742,8 +741,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Append)
.save(basePath)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build();
+ val metaClient = createMetaClient(spark, basePath)
val commits = metaClient.getActiveTimeline.filterCompletedInstants().getInstants.toArray
.map(instant => (instant.asInstanceOf[HoodieInstant]).getAction)
assertEquals(2, commits.size)
@@ -763,8 +761,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.option(DataSourceWriteOptions.OPERATION.key, DataSourceWriteOptions.INSERT_OPERATION_OPT_VAL)
.mode(SaveMode.Append)
.save(basePath)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build()
+ val metaClient = createMetaClient(spark, basePath)
val instantTime = metaClient.getActiveTimeline.filterCompletedInstants().getInstantsAsStream.findFirst().get().getTimestamp
@@ -821,8 +818,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Overwrite)
.save(basePath)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build()
+ val metaClient = createMetaClient(spark, basePath)
val commits = metaClient.getActiveTimeline.filterCompletedInstants().getInstants.toArray
.map(instant => (instant.asInstanceOf[HoodieInstant]).getAction)
assertEquals(2, commits.size)
@@ -879,8 +875,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
val filterSecondPartitionCount = recordsForPartitionColumn.filter(row => row.get(0).equals(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH)).size
assertEquals(7, filterSecondPartitionCount)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build()
+ val metaClient = createMetaClient(spark, basePath)
val commits = metaClient.getActiveTimeline.filterCompletedInstants().getInstants.toArray
.map(instant => instant.asInstanceOf[HoodieInstant].getAction)
assertEquals(3, commits.size)
@@ -933,8 +928,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
val filterSecondPartitionCount = recordsForPartitionColumn.filter(row => row.get(0).equals(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH)).size
assertEquals(7, filterSecondPartitionCount)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build()
+ val metaClient = createMetaClient(spark, basePath)
val commits = metaClient.getActiveTimeline.filterCompletedInstants().getInstants.toArray
.map(instant => instant.asInstanceOf[HoodieInstant].getAction)
assertEquals(2, commits.size)
@@ -1553,10 +1547,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.saveAsTable("hoodie_test")
// init metaClient
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
assertEquals(spark.read.format("hudi").options(readOpts).load(basePath).count(), 5)
// use the Append mode
@@ -1813,10 +1804,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
}
def assertLastCommitIsUpsert(): Boolean = {
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(hadoopConf)
- .build()
+ val metaClient = createMetaClient(basePath)
val timeline = metaClient.getActiveTimeline.getAllCommitsTimeline
val latestCommit = timeline.lastInstant()
assert(latestCommit.isPresent)
@@ -1851,10 +1839,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.mode(SaveMode.Overwrite)
.save(basePath)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(hadoopConf)
- .build()
+ val metaClient = createMetaClient(basePath)
assertFalse(metaClient.getActiveTimeline.getLastClusteringInstant.isPresent)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSourceStorage.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSourceStorage.scala
index 0807c0f9ff4..f71759a1ec6 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSourceStorage.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSourceStorage.scala
@@ -23,13 +23,13 @@ import org.apache.hudi.client.validator.{SqlQueryEqualityPreCommitValidator, Sql
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.config.TimestampKeyGeneratorConfig.{TIMESTAMP_INPUT_DATE_FORMAT, TIMESTAMP_OUTPUT_DATE_FORMAT, TIMESTAMP_TYPE_FIELD}
import org.apache.hudi.common.model.WriteOperationType
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.{HoodiePreCommitValidatorConfig, HoodieWriteConfig}
import org.apache.hudi.exception.{HoodieUpsertException, HoodieValidationException}
import org.apache.hudi.keygen.{NonpartitionedKeyGenerator, TimestampBasedKeyGenerator}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers}
@@ -37,9 +37,9 @@ import org.apache.hudi.common.fs.FSUtils
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.spark.SparkConf
-import org.apache.spark.sql._
import org.apache.spark.sql.functions.{col, lit}
import org.apache.spark.sql.types.StringType
+import org.apache.spark.sql.{DataFrame, SaveMode}
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertThrows, assertTrue}
import org.junit.jupiter.api.Tag
import org.junit.jupiter.api.function.Executable
@@ -280,8 +280,7 @@ class TestCOWDataSourceStorage extends SparkClientFunctionalTestHarness {
}
assertRecordCount(basePath, expectedRecCount + 500)
- val metaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build()
+ val metaClient = createMetaClient(spark, basePath)
val commits = metaClient.getActiveTimeline.filterCompletedInstants().getInstants.toArray
.map(instant => instant.asInstanceOf[HoodieInstant].getAction)
// assert replace commit is archived and not part of active timeline.
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala
index 2998d4facac..a5ec984d8be 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestHoodieActiveTimeline.scala
@@ -17,16 +17,17 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers}
import org.apache.hudi.common.model.HoodieFileFormat
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceWriteOptions, HoodieDataSourceHelpers}
import org.apache.spark.sql._
-import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.{AfterEach, BeforeEach, Test}
import org.slf4j.LoggerFactory
import scala.collection.JavaConversions._
@@ -84,7 +85,7 @@ class TestHoodieActiveTimeline extends HoodieSparkClientTestBase {
.map(_.get(0).toString).sorted
assert(Array("2015/03/16", "2015/03/17", "2016/03/15").sameElements(partitionsForCommit1))
- val metaClient: HoodieTableMetaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(hadoopConf).build()
+ val metaClient: HoodieTableMetaClient = createMetaClient(basePath)
var activeTimeline = metaClient.getActiveTimeline
// check that get the latest parquet file
@@ -154,7 +155,7 @@ class TestHoodieActiveTimeline extends HoodieSparkClientTestBase {
.save(basePath)
val commit1Time = HoodieDataSourceHelpers.latestCommit(storage, basePath)
- val metaClient: HoodieTableMetaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(hadoopConf).build()
+ val metaClient: HoodieTableMetaClient = createMetaClient(basePath)
var activeTimeline = metaClient.getActiveTimeline
// check that get the latest parquet file
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadByStateTransitionTime.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadByStateTransitionTime.scala
index a5718d05921..2efd5e08257 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadByStateTransitionTime.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadByStateTransitionTime.scala
@@ -17,14 +17,13 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.HoodieTableType
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.TimelineUtils.HollowCommitHandling.USE_TRANSITION_TIME
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.spark.sql.{SaveMode, SparkSession}
import org.junit.jupiter.api.{AfterEach, Assertions, BeforeEach}
@@ -76,11 +75,7 @@ class TestIncrementalReadByStateTransitionTime extends HoodieSparkClientTestBase
.mode(SaveMode.Append)
.save(basePath)
- val metaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true)
- .build()
+ val metaClient = createMetaClient(spark, basePath)
val firstInstant = metaClient.getActiveTimeline.filterCompletedInstants().getInstantsOrderedByStateTransitionTime
.findFirst().get()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFullTableScan.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFullTableScan.scala
index e26c9954470..3e44b015b18 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFullTableScan.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFullTableScan.scala
@@ -17,23 +17,22 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.HoodieTableType
-import org.apache.hudi.common.table.HoodieTableMetaClient
-import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.table.timeline.HoodieTimeline.GREATER_THAN
+import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieInstantTimeGenerator, HoodieTimeline}
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.HoodieIncrementalPathNotFoundException
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.spark.SparkException
import org.apache.spark.sql.{SaveMode, SparkSession}
-import org.junit.jupiter.api.{AfterEach, BeforeEach}
import org.junit.jupiter.api.Assertions.{assertEquals, assertThrows, assertTrue}
import org.junit.jupiter.api.function.Executable
+import org.junit.jupiter.api.{AfterEach, BeforeEach}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
@@ -88,7 +87,7 @@ class TestIncrementalReadWithFullTableScan extends HoodieSparkClientTestBase {
.save(basePath)
}
- val hoodieMetaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build()
+ val hoodieMetaClient = createMetaClient(spark, basePath)
/**
* State of timeline after 10 commits
* +------------------+--------------------------------------+
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala
index 565f68e44fd..8475e6c2e95 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestLayoutOptimization.scala
@@ -18,22 +18,22 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.hudi.HoodieFileIndex.DataSkippingFailureMode
import org.apache.hudi.common.config.HoodieMetadataConfig
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieInstant, HoodieTimeline}
+import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.config.{HoodieClusteringConfig, HoodieWriteConfig}
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
import org.apache.spark.sql._
import org.apache.spark.sql.types._
-import org.junit.jupiter.api.{AfterEach, BeforeEach, Tag}
import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.{AfterEach, BeforeEach, Tag}
import org.junit.jupiter.params.ParameterizedTest
-import org.junit.jupiter.params.provider.{Arguments, MethodSource}
import org.junit.jupiter.params.provider.Arguments.arguments
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
import scala.collection.JavaConversions._
@@ -120,11 +120,7 @@ class TestLayoutOptimization extends HoodieSparkClientTestBase {
.mode(SaveMode.Overwrite)
.save(basePath)
- val hudiMetaClient = HoodieTableMetaClient.builder
- .setConf(hadoopConf)
- .setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true)
- .build
+ val hudiMetaClient = createMetaClient(basePath)
val lastCommit = hudiMetaClient.getActiveTimeline.getAllCommitsTimeline.lastInstant().get()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
index 0f9a7bcbe04..472a706324c 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
@@ -50,7 +50,6 @@ import org.junit.jupiter.params.provider.{CsvSource, EnumSource, ValueSource}
import org.slf4j.LoggerFactory
import java.util.function.Consumer
-
import scala.collection.JavaConversions.mapAsJavaMap
import scala.collection.JavaConverters._
@@ -1156,10 +1155,7 @@ class TestMORDataSource extends HoodieSparkClientTestBase with SparkDatasetMixin
.options(options)
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val commit1Time = metaClient.getActiveTimeline.lastInstant().get().getTimestamp
val dataGen2 = new HoodieTestDataGenerator(Array("2022-01-02"))
@@ -1423,10 +1419,7 @@ class TestMORDataSource extends HoodieSparkClientTestBase with SparkDatasetMixin
.option(DataSourceWriteOptions.RECORD_MERGER_STRATEGY.key(), mergerStrategyName)
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
assertEquals(metaClient.getTableConfig.getRecordMergerStrategy, mergerStrategyName)
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
index 32b188aa7d0..f45ac02811e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSourceStorage.scala
@@ -20,16 +20,15 @@
package org.apache.hudi.functional
import org.apache.hudi.common.config.HoodieMetadataConfig
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
+import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestUtils}
import org.apache.hudi.common.util.StringUtils
import org.apache.hudi.config.HoodieWriteConfig
+import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers}
-import org.apache.hudi.common.fs.FSUtils
-import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql._
@@ -177,8 +176,7 @@ class TestMORDataSourceStorage extends SparkClientFunctionalTestHarness {
.save(basePath)
}
// compaction should have been completed
- val metaClient = HoodieTableMetaClient.builder.setConf(fs.getConf).setBasePath(basePath)
- .setLoadActiveTimelineOnLoad(true).build
+ val metaClient = HoodieTestUtils.createMetaClient(fs.getConf, basePath)
assertEquals(1, metaClient.getActiveTimeline.getCommitTimeline.countInstants())
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestParquetColumnProjection.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestParquetColumnProjection.scala
index ee1edbcccb2..0173c3f642a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestParquetColumnProjection.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestParquetColumnProjection.scala
@@ -17,17 +17,19 @@
package org.apache.hudi.functional
-import org.apache.avro.Schema
-import org.apache.calcite.runtime.SqlFunctions.abs
import org.apache.hudi.HoodieBaseRelation.projectSchema
import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieStorageConfig}
import org.apache.hudi.common.model.{HoodieRecord, OverwriteNonDefaultsWithLatestAvroPayload}
-import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
+import org.apache.hudi.common.table.HoodieTableConfig
import org.apache.hudi.common.testutils.{HadoopMapRedUtils, HoodieTestDataGenerator}
import org.apache.hudi.config.{HoodieCompactionConfig, HoodieWriteConfig}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness.getSparkSqlConf
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, DefaultSource, HoodieBaseRelation, HoodieSparkUtils, HoodieUnsafeRDD}
+
+import org.apache.avro.Schema
+import org.apache.calcite.runtime.SqlFunctions.abs
import org.apache.parquet.hadoop.util.counters.BenchmarkCounter
import org.apache.spark.SparkConf
import org.apache.spark.internal.Logging
@@ -310,7 +312,7 @@ class TestParquetColumnProjection extends SparkClientFunctionalTestHarness with
* | updated data | 001 | 002 | 003 | | 004 | 005 | 006 |
* +--------------+--------------+--------------+--------------+--------------------+--------------+--------------+--------------+
*/
- val hoodieMetaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(tablePath).setLoadActiveTimelineOnLoad(true).build()
+ val hoodieMetaClient = createMetaClient(spark, tablePath)
val completedCommits = hoodieMetaClient.getCommitsAndCompactionTimeline.filterCompletedInstants()
val startUnarchivedCommitTs = completedCommits.nthInstant(1).get().getTimestamp //deltacommit2
val endUnarchivedCommitTs = completedCommits.nthInstant(5).get().getTimestamp //deltacommit6
@@ -336,7 +338,7 @@ class TestParquetColumnProjection extends SparkClientFunctionalTestHarness with
bootstrapMORTableWithDeltaLog(tablePath, targetRecordsCount, defaultWriteOpts, populateMetaFields = true, inlineCompact = true)
- val hoodieMetaClient = HoodieTableMetaClient.builder().setConf(spark.sparkContext.hadoopConfiguration).setBasePath(tablePath).setLoadActiveTimelineOnLoad(true).build()
+ val hoodieMetaClient = createMetaClient(spark, tablePath)
val completedCommits = hoodieMetaClient.getCommitsAndCompactionTimeline.filterCompletedInstants()
val startUnarchivedCommitTs = (completedCommits.nthInstant(1).get().getTimestamp.toLong - 1L).toString
val endUnarchivedCommitTs = completedCommits.nthInstant(3).get().getTimestamp //commit
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
index 80d151d5b5e..b5c487b6bca 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlCoreFlow.scala
@@ -24,13 +24,14 @@ import org.apache.hudi.HoodieDataSourceHelpers.{hasNewCommits, latestCommit, lis
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.WriteOperationType.{BULK_INSERT, INSERT, UPSERT}
import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.TimelineUtils
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.keygen.NonpartitionedKeyGenerator
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.{DataSourceReadOptions, HoodieSparkUtils}
+
import org.apache.spark.sql
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import org.apache.spark.sql.{Dataset, Row}
@@ -229,10 +230,7 @@ class TestSparkSqlCoreFlow extends HoodieSparkSqlTestBase {
}
def assertOperation(basePath: String, count: Int, operationType: WriteOperationType): Boolean = {
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, basePath)
val timeline = metaClient.getActiveTimeline.getAllCommitsTimeline
assert(timeline.countInstants() == count)
val latestCommit = timeline.lastInstant()
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
index 0fc2976c7fe..12b4c32bbaa 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestSparkSqlWithCustomKeyGenerator.scala
@@ -21,11 +21,12 @@ package org.apache.hudi.functional
import org.apache.hudi.HoodieSparkUtils
import org.apache.hudi.common.config.TypedProperties
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.util.StringUtils
import org.apache.hudi.exception.HoodieException
import org.apache.hudi.functional.TestSparkSqlWithCustomKeyGenerator._
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.util.SparkKeyGenUtils
+
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import org.joda.time.DateTime
@@ -456,10 +457,7 @@ class TestSparkSqlWithCustomKeyGenerator extends HoodieSparkSqlTestBase {
.save(tablePath)
// Validate that the generated table has expected table configs of key generator and partition path fields
- val metaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath)
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
assertEquals(keyGenClassName, metaClient.getTableConfig.getKeyGeneratorClassName)
// Validate that that partition path fields in the table config should always
// contain the field names only (no key generator type like "segment:simple")
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala
index 9e6663ea75c..fe3278fb751 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala
@@ -17,7 +17,6 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers}
import org.apache.hudi.DataSourceWriteOptions.STREAMING_CHECKPOINT_IDENTIFIER
import org.apache.hudi.HoodieStreamingSink.SINK_CHECKPOINT_KEY
import org.apache.hudi.client.transaction.lock.InProcessLockProvider
@@ -25,28 +24,29 @@ import org.apache.hudi.common.config.HoodieStorageConfig
import org.apache.hudi.common.model.{FileSlice, HoodieTableType, WriteConcurrencyMode}
import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieTimeline
-import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestTable}
import org.apache.hudi.common.testutils.RawTripTestPayload.recordsToStrings
+import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestTable, HoodieTestUtils}
import org.apache.hudi.common.util.{CollectionUtils, CommitUtils}
import org.apache.hudi.config.{HoodieClusteringConfig, HoodieCompactionConfig, HoodieLockConfig, HoodieWriteConfig}
import org.apache.hudi.exception.TableNotFoundException
-import org.apache.hudi.storage.{StoragePath, HoodieStorage}
+import org.apache.hudi.storage.{HoodieStorage, StoragePath}
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, HoodieDataSourceHelpers}
import org.apache.hadoop.conf.Configuration
import org.apache.spark.sql._
import org.apache.spark.sql.streaming.{OutputMode, StreamingQuery, Trigger}
import org.apache.spark.sql.types.StructType
-import org.junit.jupiter.api.{BeforeEach, Test}
import org.junit.jupiter.api.Assertions.{assertEquals, assertTrue}
+import org.junit.jupiter.api.{BeforeEach, Test}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{EnumSource, ValueSource}
import org.slf4j.LoggerFactory
import scala.collection.JavaConversions._
-import scala.concurrent.{Await, Future}
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.duration.Duration
+import scala.concurrent.{Await, Future}
/**
* Basic tests on the spark datasource for structured streaming sink
@@ -288,9 +288,7 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase {
.start(destPath)
query1.processAllAvailable()
- var metaClient = HoodieTableMetaClient.builder
- .setConf(storage.getConf.asInstanceOf[Configuration])
- .setBasePath(destPath).setLoadActiveTimelineOnLoad(true).build
+ var metaClient = HoodieTestUtils.createMetaClient(storage, destPath)
assertLatestCheckpointInfoMatched(metaClient, "streaming_identifier1", "0")
@@ -335,9 +333,7 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase {
query3.processAllAvailable()
query3.stop()
- metaClient = HoodieTableMetaClient.builder
- .setConf(storage.getConf.asInstanceOf[Configuration])
- .setBasePath(destPath).setLoadActiveTimelineOnLoad(true).build
+ metaClient = HoodieTestUtils.createMetaClient(storage, destPath)
assertLatestCheckpointInfoMatched(metaClient, "streaming_identifier1", "2")
assertLatestCheckpointInfoMatched(metaClient, "streaming_identifier2", "0")
@@ -372,9 +368,7 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase {
.start(destPath)
query1.processAllAvailable()
- val metaClient = HoodieTableMetaClient.builder
- .setConf(storage.getConf.asInstanceOf[Configuration])
- .setBasePath(destPath).setLoadActiveTimelineOnLoad(true).build
+ val metaClient = HoodieTestUtils.createMetaClient(storage, destPath)
assertLatestCheckpointInfoMatched(metaClient, STREAMING_CHECKPOINT_IDENTIFIER.defaultValue(), "0")
query1.stop()
@@ -416,10 +410,7 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase {
currNumCommits = waitTillAtleastNCommits(storage, destPath, currNumCommits + 1, 120, 5)
// Wait for the clustering to finish
- this.metaClient = HoodieTableMetaClient.builder()
- .setConf(storage.getConf.asInstanceOf[Configuration])
- .setBasePath(destPath)
- .setLoadActiveTimelineOnLoad(true).build()
+ this.metaClient = HoodieTestUtils.createMetaClient(storage, destPath)
checkClusteringResult(destPath)
assertEquals(3, HoodieDataSourceHelpers.listCommitsSince(storage, destPath, "000").size())
@@ -473,9 +464,7 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase {
}
private def latestInstant(storage: HoodieStorage, basePath: String, instantAction: String): String = {
- val metaClient = HoodieTableMetaClient.builder
- .setConf(storage.getConf.asInstanceOf[Configuration])
- .setBasePath(basePath).setLoadActiveTimelineOnLoad(true).build
+ val metaClient = HoodieTestUtils.createMetaClient(storage, basePath)
metaClient.getActiveTimeline
.getTimelineOfActions(CollectionUtils.createSet(instantAction))
.filterCompletedInstants
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
index 367d9998759..504d7a53aac 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
@@ -17,22 +17,22 @@
package org.apache.hudi.functional
-import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, ScalaAssertionSupport}
import org.apache.hudi.common.config.HoodieMetadataConfig
-import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieTableType}
import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, MERGE_ON_READ}
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
+import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieTableType}
+import org.apache.hudi.common.table.TableSchemaResolver
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline
import org.apache.hudi.common.testutils.HoodieTestTable
import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig, HoodieCompactionConfig, HoodieWriteConfig}
import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.exception.HoodieTimeTravelException
import org.apache.hudi.testutils.HoodieSparkClientTestBase
+import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions, ScalaAssertionSupport}
-import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.apache.spark.sql.SaveMode.{Append, Overwrite}
-import org.junit.jupiter.api.{AfterEach, BeforeEach}
+import org.apache.spark.sql.{DataFrame, Row, SaveMode, SparkSession}
import org.junit.jupiter.api.Assertions.{assertEquals, assertNotNull, assertNull, assertTrue}
+import org.junit.jupiter.api.{AfterEach, BeforeEach}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.EnumSource
@@ -252,10 +252,7 @@ class TestTimeTravelQuery extends HoodieSparkClientTestBase with ScalaAssertionS
val _spark = spark
import _spark.implicits._
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val opts = commonOpts ++ Map(
DataSourceWriteOptions.TABLE_TYPE.key -> tableType.name,
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
index 210ea00048e..efde9296406 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCDataFrameSuite.scala
@@ -18,19 +18,20 @@
package org.apache.hudi.functional.cdc
-import org.apache.avro.generic.GenericRecord
import org.apache.hudi.DataSourceWriteOptions
import org.apache.hudi.DataSourceWriteOptions.{MOR_TABLE_TYPE_OPT_VAL, PARTITIONPATH_FIELD_OPT_KEY, PRECOMBINE_FIELD_OPT_KEY, RECORDKEY_FIELD_OPT_KEY}
import org.apache.hudi.QuickstartUtils.getQuickstartWriteConfigs
import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode.OP_KEY_ONLY
import org.apache.hudi.common.table.cdc.HoodieCDCUtils.schemaBySupplementalLoggingMode
import org.apache.hudi.common.table.cdc.{HoodieCDCOperation, HoodieCDCSupplementalLoggingMode}
-import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver}
+import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver}
import org.apache.hudi.common.testutils.HoodieTestDataGenerator
import org.apache.hudi.common.testutils.RawTripTestPayload.{deleteRecordsToStrings, recordsToStrings}
import org.apache.hudi.config.HoodieWriteConfig
-import org.apache.spark.sql.{Row, SaveMode}
+
+import org.apache.avro.generic.GenericRecord
import org.apache.spark.sql.types.{StringType, StructField, StructType}
+import org.apache.spark.sql.{Row, SaveMode}
import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.{CsvSource, EnumSource}
@@ -69,10 +70,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val schemaResolver = new TableSchemaResolver(metaClient)
val dataSchema = schemaResolver.getTableAvroSchema(false)
@@ -262,10 +260,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val schemaResolver = new TableSchemaResolver(metaClient)
val dataSchema = schemaResolver.getTableAvroSchema(false)
@@ -491,10 +486,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
assert(partitionToCnt.contains(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH))
// init meta client
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
totalInsertedCnt += 100
val instant1 = metaClient.reloadActiveTimeline.lastInstant().get()
@@ -602,10 +594,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
val schemaResolver = new TableSchemaResolver(metaClient)
val dataSchema = schemaResolver.getTableAvroSchema(false)
@@ -717,10 +706,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.mode(SaveMode.Overwrite)
.save(basePath)
- metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf)
- .build()
+ metaClient = createMetaClient(spark, basePath)
// Upsert Operation
val hoodieRecords2 = dataGen.generateUniqueUpdates("001", 50)
@@ -809,11 +795,7 @@ class TestCDCDataFrameSuite extends HoodieCDCTestBase {
.option("hoodie.table.cdc.supplemental.logging.mode", loggingMode.name())
.mode(SaveMode.Append).save(basePath)
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(hadoopConf)
- .build()
+ val metaClient = createMetaClient(spark, basePath)
val startTimeStamp = metaClient.reloadActiveTimeline().firstInstant().get.getTimestamp
val latestTimeStamp = metaClient.reloadActiveTimeline().lastInstant().get.getTimestamp
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
index 28a993e0510..947d6263663 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/cdc/TestCDCStreamingSuite.scala
@@ -17,10 +17,11 @@
package org.apache.hudi.functional.cdc
+import org.apache.hudi.common.table.HoodieTableConfig
import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode
-import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.{DataSourceReadOptions, DataSourceWriteOptions}
+
import org.apache.spark.sql.QueryTest.checkAnswer
import org.apache.spark.sql.catalyst.expressions.{Add, If, Literal}
import org.apache.spark.sql.execution.streaming.MemoryStream
@@ -85,11 +86,7 @@ class TestCDCStreamingSuite extends HoodieCDCTestBase {
.option(HoodieWriteConfig.TBL_NAME.key, "country_to_population")
.save(countryToPopulationTblPath)
- val hadoopConf = spark.sessionState.newHadoopConf()
- val userToCountryMetaClient = HoodieTableMetaClient.builder()
- .setBasePath(userToCountryTblPath)
- .setConf(hadoopConf)
- .build()
+ val userToCountryMetaClient = createMetaClient(spark, userToCountryTblPath)
val inputData = new MemoryStream[(Int, String, String)](100, spark.sqlContext)
val df = inputData.toDS().toDF("userid", "country", "ts")
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
index b101e838c84..b48e4f4cb1a 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
@@ -17,18 +17,18 @@
package org.apache.spark.sql.hudi.common
-import org.apache.hadoop.fs.Path
import org.apache.hudi.HoodieSparkRecordMerger
import org.apache.hudi.common.config.HoodieStorageConfig
import org.apache.hudi.common.model.HoodieAvroRecordMerger
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.TimelineMetadataUtils
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.index.inmemory.HoodieInMemoryHashIndex
-import org.apache.hudi.testutils.HoodieClientTestUtils.getSparkConfForTest
+import org.apache.hudi.testutils.HoodieClientTestUtils.{createMetaClient, getSparkConfForTest}
+
+import org.apache.hadoop.fs.Path
import org.apache.spark.SparkConf
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase.checkMessageContains
@@ -233,19 +233,13 @@ class HoodieSparkSqlTestBase extends FunSuite with BeforeAndAfterAll {
object HoodieSparkSqlTestBase {
def getLastCommitMetadata(spark: SparkSession, tablePath: String) = {
- val metaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath)
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
metaClient.getActiveTimeline.getLastCommitMetadataWithValidData.get.getRight
}
def getLastCleanMetadata(spark: SparkSession, tablePath: String) = {
- val metaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath)
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val cleanInstant = metaClient.reloadActiveTimeline().getCleanerTimeline.filterCompletedInstants().lastInstant().get()
TimelineMetadataUtils.deserializeHoodieCleanMetadata(metaClient
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala
index 0b391229c2f..6b546aca921 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/TestSqlConf.scala
@@ -22,6 +22,7 @@ import org.apache.hudi.common.config.DFSPropertiesConfiguration
import org.apache.hudi.common.model.HoodieTableType
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
import org.apache.hudi.storage.HoodieStorageUtils
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hadoop.conf.Configuration
import org.scalatest.BeforeAndAfter
@@ -64,10 +65,7 @@ class TestSqlConf extends HoodieSparkSqlTestBase with BeforeAndAfter {
// First insert a new record
spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000, $partitionVal)")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val firstCommit = metaClient.getActiveTimeline.filterCompletedInstants().lastInstant().get().getTimestamp
// Then insert another new record
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
index 268f5a87bc1..0db0d8f761c 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTable.scala
@@ -19,7 +19,9 @@ package org.apache.spark.sql.hudi.ddl
import org.apache.hudi.HoodieSparkUtils
import org.apache.hudi.common.model.HoodieRecord
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
+import org.apache.hudi.common.table.TableSchemaResolver
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
@@ -79,9 +81,7 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
spark.sessionState.catalog.tableExists(new TableIdentifier(newTableName))
)
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(tablePath)
- .setConf(hadoopConf).build()
+ val metaClient = createMetaClient(spark, tablePath)
assertResult(newTableName) (metaClient.getTableConfig.getTableName)
validateTableSchema(tablePath)
@@ -215,10 +215,7 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
}
def validateTableSchema(tablePath: String): Unit = {
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(tablePath)
- .setConf(hadoopConf).build()
-
+ val metaClient = createMetaClient(spark, tablePath)
val schema = new TableSchemaResolver(metaClient).getTableAvroSchema(false)
assertFalse(schema.getFields.asScala.exists(f => HoodieRecord.HOODIE_META_COLUMNS.contains(f.name())),
"Metadata fields should be excluded from the table schema")
@@ -348,9 +345,7 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
spark.sessionState.catalog.tableExists(new TableIdentifier(newTableName))
)
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(tablePath)
- .setConf(hadoopConf).build()
+ val metaClient = createMetaClient(spark, tablePath)
assertResult(newTableName) (metaClient.getTableConfig.getTableName)
// insert some data
@@ -415,10 +410,7 @@ class TestAlterTable extends HoodieSparkSqlTestBase {
spark.sql(s"alter table $tableName add columns(ext0 string)")
}
- val metaClient = HoodieTableMetaClient.builder
- .setConf(spark.sqlContext.sessionState.newHadoopConf())
- .setBasePath(tablePath)
- .build
+ val metaClient = createMetaClient(spark, tablePath)
val cnt = metaClient.getActiveTimeline.countInstants()
if (cleanEnable) {
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
index f2126da5872..bdaf51e9bd2 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestAlterTableDropPartition.scala
@@ -20,11 +20,11 @@ package org.apache.spark.sql.hudi.ddl
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.avro.model.{HoodieCleanMetadata, HoodieCleanPartitionMetadata}
import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieCommitMetadata}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstant}
import org.apache.hudi.common.util.{PartitionPathEncodeUtils, StringUtils, Option => HOption}
import org.apache.hudi.config.{HoodieCleanConfig, HoodieWriteConfig}
import org.apache.hudi.keygen.{ComplexKeyGenerator, SimpleKeyGenerator}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.{HoodieCLIUtils, HoodieSparkUtils}
import org.apache.spark.sql.SaveMode
@@ -474,9 +474,7 @@ class TestAlterTableDropPartition extends HoodieSparkSqlTestBase {
)
// check schema
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(s"${tmp.getCanonicalPath}/$tableName")
- .setConf(hadoopConf).build()
+ val metaClient = createMetaClient(spark, s"${tmp.getCanonicalPath}/$tableName")
val lastInstant = metaClient.getActiveTimeline.getCommitsTimeline.lastInstant()
val commitMetadata = HoodieCommitMetadata.fromBytes(metaClient.getActiveTimeline.getInstantDetails(
lastInstant.get()).get(), classOf[HoodieCommitMetadata])
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
index 0d757f4bedb..313cbf895b9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestCreateTable.scala
@@ -20,11 +20,13 @@ package org.apache.spark.sql.hudi.ddl
import org.apache.hudi.DataSourceWriteOptions._
import org.apache.hudi.HoodieSparkUtils
import org.apache.hudi.common.model.{HoodieRecord, WriteOperationType}
-import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient}
+import org.apache.hudi.common.table.HoodieTableConfig
import org.apache.hudi.common.util.PartitionPathEncodeUtils.escapePathName
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat
import org.apache.hudi.keygen.SimpleKeyGenerator
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.catalyst.catalog.{CatalogTableType, HoodieCatalogTable}
@@ -78,10 +80,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertFalse(table.properties.contains(OPERATION.key()))
val tablePath = table.storage.properties("path")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val tableConfig = metaClient.getTableConfig
assertResult(databaseName)(tableConfig.getDatabaseName)
assertResult(tableName)(tableConfig.getTableName)
@@ -136,10 +135,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertFalse(table.properties.contains(OPERATION.key()))
val tablePath = table.storage.properties("path")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val tableConfig = metaClient.getTableConfig.getProps.asScala.toMap
assertResult(true)(tableConfig.contains(HoodieTableConfig.CREATE_SCHEMA.key))
assertResult("dt")(tableConfig(HoodieTableConfig.PARTITION_FIELDS.key))
@@ -797,10 +793,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertFalse(table.properties.contains(OPERATION.key()))
val tablePath = table.storage.properties("path")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val tableConfig = metaClient.getTableConfig.getProps.asScala.toMap
assertResult("default")(tableConfig(HoodieTableConfig.DATABASE_NAME.key()))
assertResult(tableName)(tableConfig(HoodieTableConfig.NAME.key()))
@@ -836,10 +829,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
assertFalse(table.properties.contains(OPERATION.key()))
val tablePath = table.storage.properties("path")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val tableConfig = metaClient.getTableConfig.getProps.asScala.toMap
assertResult("default")(tableConfig(HoodieTableConfig.DATABASE_NAME.key()))
assertResult(tableName)(tableConfig(HoodieTableConfig.NAME.key()))
@@ -916,10 +906,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
Seq(1, "a1", 10, 1000, partitionValue)
)
// Check the missing properties for spark sql
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val properties = metaClient.getTableConfig.getProps.asScala.toMap
assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key))
assertResult("dt")(properties(HoodieTableConfig.PARTITION_FIELDS.key))
@@ -990,10 +977,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
Seq(1, "a1", 10, 1000, day, 12)
)
// Check the missing properties for spark sql
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val properties = metaClient.getTableConfig.getProps.asScala.toMap
assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key))
assertResult("day,hh")(properties(HoodieTableConfig.PARTITION_FIELDS.key))
@@ -1061,10 +1045,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
Seq(1, "a1", 10, 1000)
)
// Check the missing properties for spark sql
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tmp.getCanonicalPath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath)
val properties = metaClient.getTableConfig.getProps.asScala.toMap
assertResult(true)(properties.contains(HoodieTableConfig.CREATE_SCHEMA.key))
assertResult("ts")(properties(HoodieTableConfig.PRECOMBINE_FIELD.key))
@@ -1203,10 +1184,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
test("Test Infer KegGenClazz") {
def checkKeyGenerator(targetGenerator: String, tableName: String) = {
val tablePath = spark.sessionState.catalog.getTableMetadata(TableIdentifier(tableName)).location.getPath
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
val realKeyGenerator =
metaClient.getTableConfig.getProps.asScala.toMap.get(HoodieTableConfig.KEY_GENERATOR_CLASS_NAME.key).get
assertResult(targetGenerator)(realKeyGenerator)
@@ -1385,9 +1363,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
// drop the table without purging hdfs directory
spark.sql(s"drop table $tableName".stripMargin)
- val tableSchemaAfterCreate1 = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath).build().getTableConfig.getTableCreateSchema
+ val tableSchemaAfterCreate1 = createMetaClient(spark, tablePath).getTableConfig.getTableCreateSchema
// avro schema name and namespace should not change should not change
spark.newSession().sql(
@@ -1406,9 +1382,7 @@ class TestCreateTable extends HoodieSparkSqlTestBase {
| )
""".stripMargin)
- val tableSchemaAfterCreate2 = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tablePath).build().getTableConfig.getTableCreateSchema
+ val tableSchemaAfterCreate2 = createMetaClient(spark, tablePath).getTableConfig.getTableCreateSchema
assertResult(tableSchemaAfterCreate1.get)(tableSchemaAfterCreate2.get)
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
index 5e43d714a5e..d3a2270d622 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/ddl/TestSpark3DDL.scala
@@ -17,16 +17,18 @@
package org.apache.spark.sql.hudi.ddl
-import org.apache.hadoop.fs.Path
import org.apache.hudi.common.config.HoodieStorageConfig
import org.apache.hudi.common.model.HoodieRecord
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
-import org.apache.hudi.common.table.{HoodieTableMetaClient, TableSchemaResolver}
+import org.apache.hudi.common.table.TableSchemaResolver
import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, RawTripTestPayload}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.index.inmemory.HoodieInMemoryHashIndex
import org.apache.hudi.testutils.DataSourceTestUtils
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.{DataSourceWriteOptions, HoodieSparkRecordMerger, HoodieSparkUtils, QuickstartUtils}
+
+import org.apache.hadoop.fs.Path
import org.apache.spark.sql.catalyst.TableIdentifier
import org.apache.spark.sql.functions.{arrays_zip, col, expr, lit}
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
@@ -458,8 +460,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
}
private def validateInternalSchema(basePath: String, isDropColumn: Boolean, currentMaxColumnId: Int): Unit = {
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(hadoopConf).build()
+ val metaClient = createMetaClient(spark, basePath)
val schema = new TableSchemaResolver(metaClient).getTableInternalSchemaFromCommitMetadata.get()
val lastInstant = metaClient.getActiveTimeline.filterCompletedInstants().lastInstant().get()
assert(schema.schemaId() == lastInstant.getTimestamp.toLong)
@@ -471,8 +472,7 @@ class TestSpark3DDL extends HoodieSparkSqlTestBase {
}
private def getMaxColumnId(basePath: String): Int = {
- val hadoopConf = spark.sessionState.newHadoopConf()
- val metaClient = HoodieTableMetaClient.builder().setBasePath(basePath).setConf(hadoopConf).build()
+ val metaClient = createMetaClient(spark, basePath)
new TableSchemaResolver(metaClient).getTableInternalSchemaFromCommitMetadata.get.getMaxColumnId
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestCDCForSparkSQL.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestCDCForSparkSQL.scala
index 59f9eed83b0..e55bab0d33c 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestCDCForSparkSQL.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestCDCForSparkSQL.scala
@@ -19,8 +19,9 @@ package org.apache.spark.sql.hudi.dml
import org.apache.hudi.DataSourceReadOptions._
import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_INSERT_INTO_OPERATION
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.cdc.HoodieCDCSupplementalLoggingMode.{DATA_BEFORE, DATA_BEFORE_AFTER, OP_KEY_ONLY}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.functions._
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
@@ -71,10 +72,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| )
| location '$basePath'
""".stripMargin)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, basePath)
spark.sql(s"insert into $tableName values (1, 11, 1000, 'a1'), (2, 12, 1000, 'a2')")
assert(spark.sql(s"select _hoodie_file_name from $tableName").distinct().count() == 2)
val fgForID1 = spark.sql(s"select _hoodie_file_name from $tableName where id=1").head().get(0)
@@ -129,10 +127,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| location '$basePath'
""".stripMargin)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, basePath)
spark.sql(s"insert into $tableName values (1, 'a1', 11, 1000), (2, 'a2', 12, 1000), (3, 'a3', 13, 1000)")
val commitTime1 = metaClient.reloadActiveTimeline.lastInstant().get().getTimestamp
@@ -254,10 +249,7 @@ class TestCDCForSparkSQL extends HoodieSparkSqlTestBase {
| location '$basePath'
""".stripMargin)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(basePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, basePath)
spark.sql(
s"""
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
index 3290c099a9c..431f042bf22 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestInsertTable.scala
@@ -27,7 +27,10 @@ import org.apache.hudi.config.{HoodieClusteringConfig, HoodieIndexConfig, Hoodie
import org.apache.hudi.exception.{HoodieDuplicateKeyException, HoodieException}
import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode
import org.apache.hudi.index.HoodieIndex.IndexType
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.{DataSourceWriteOptions, HoodieCLIUtils, HoodieSparkUtils}
+
+import org.apache.spark.scheduler.{SparkListener, SparkListenerStageSubmitted}
import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.hudi.HoodieSqlCommonUtils
import org.apache.spark.sql.hudi.command.HoodieSparkValidateDuplicateKeyRecordMerger
@@ -214,10 +217,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
| select 20 as price, 2000 as ts, 2 as id, 'a2' as name
""".stripMargin)
// should not mess with the original order after write the out-of-order data.
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tmp.getCanonicalPath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath)
val schema = HoodieSqlCommonUtils.getTableSqlSchema(metaClient).get
assert(schema.getFieldIndex("id").contains(0))
assert(schema.getFieldIndex("price").contains(2))
@@ -262,10 +262,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
| select 1 as id, '2021-01-05' as dt, 'a1' as name, 10 as price, 1000 as ts
""".stripMargin)
// should not mess with the original order after write the out-of-order data.
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tmp.getCanonicalPath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath)
val schema = HoodieSqlCommonUtils.getTableSqlSchema(metaClient).get
assert(schema.getFieldIndex("id").contains(0))
assert(schema.getFieldIndex("price").contains(2))
@@ -768,10 +765,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
checkAnswer(s"select id, name, price from $tableName")(
Seq(1, "a1", 10.0)
)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tmp.getCanonicalPath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath)
assertResult(tableName)(metaClient.getTableConfig.getTableName)
}
}
@@ -1323,10 +1317,7 @@ class TestInsertTable extends HoodieSparkSqlTestBase {
.mode(SaveMode.Overwrite)
.save(tablePath)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tablePath)
assertResult(true)(new TableSchemaResolver(metaClient).hasOperationField)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable2.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable2.scala
index f58935b5bf3..0ed43aa8f48 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable2.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestMergeIntoTable2.scala
@@ -18,7 +18,8 @@
package org.apache.spark.sql.hudi.dml
import org.apache.hudi.HoodieSparkUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.Row
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
@@ -155,10 +156,7 @@ class TestMergeIntoTable2 extends HoodieSparkSqlTestBase {
|select 1 as id, 'a1' as name
|""".stripMargin
)
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(tmp.getCanonicalPath)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath)
// check record key in hoodie.properties
assertResult("id")(metaClient.getTableConfig.getRecordKeyFields.get().mkString(","))
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
index 9924b700353..183480fe691 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestTimeTravelTable.scala
@@ -18,7 +18,8 @@
package org.apache.spark.sql.hudi.dml
import org.apache.hudi.HoodieSparkUtils
-import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
class TestTimeTravelTable extends HoodieSparkSqlTestBase {
@@ -45,10 +46,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
// 1st commit instant
spark.sql(s"insert into $tableName1 values(1, 'a1', 10, 1000)")
- val metaClient1 = HoodieTableMetaClient.builder()
- .setBasePath(s"${tmp.getCanonicalPath}/$tableName1")
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient1 = createMetaClient(spark, s"${tmp.getCanonicalPath}/$tableName1")
val instant1 = metaClient1.getActiveTimeline.getAllCommitsTimeline
.lastInstant().get().getTimestamp
@@ -91,10 +89,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
spark.sql(s"insert into $tableName1 values(1, 'a1', 10, 1000)")
- val metaClient1 = HoodieTableMetaClient.builder()
- .setBasePath(s"${tmp.getCanonicalPath}/$tableName1")
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient1 = createMetaClient(spark, s"${tmp.getCanonicalPath}/$tableName1")
val instant1 = metaClient1.getActiveTimeline.getAllCommitsTimeline
.lastInstant().get().getTimestamp
@@ -203,15 +198,8 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
Seq(4, "a4", 20.0, 1000)
)
- val metaClient1 = HoodieTableMetaClient.builder()
- .setBasePath(path1)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
-
- val metaClient2 = HoodieTableMetaClient.builder()
- .setBasePath(path2)
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient1 = createMetaClient(spark, path1)
+ val metaClient2 = createMetaClient(spark, path2)
val instant1 = metaClient1.getActiveTimeline.getAllCommitsTimeline
.lastInstant().get().getTimestamp
@@ -271,10 +259,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
// 1st commit instant
spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(s"${tmp.getCanonicalPath}/$tableName")
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, s"${tmp.getCanonicalPath}/$tableName")
val instant1 = metaClient.getActiveTimeline.getAllCommitsTimeline
.lastInstant().get().getTimestamp
@@ -316,10 +301,7 @@ class TestTimeTravelTable extends HoodieSparkSqlTestBase {
spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
- val metaClient = HoodieTableMetaClient.builder()
- .setBasePath(s"${tmp.getCanonicalPath}/$tableName")
- .setConf(spark.sessionState.newHadoopConf())
- .build()
+ val metaClient = createMetaClient(spark, s"${tmp.getCanonicalPath}/$tableName")
val instant1 = metaClient.reloadActiveTimeline().getAllCommitsTimeline
.lastInstant().get().getTimestamp
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestUpdateTable.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestUpdateTable.scala
index 5d023b8d856..8bdfe258bb7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestUpdateTable.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/dml/TestUpdateTable.scala
@@ -20,7 +20,8 @@ package org.apache.spark.sql.hudi.dml
import org.apache.hudi.DataSourceWriteOptions.SPARK_SQL_OPTIMIZED_WRITES
import org.apache.hudi.HoodieSparkUtils.isSpark2
import org.apache.hudi.common.model.HoodieTableType
-import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
+
import org.apache.spark.sql.hudi.common.HoodieSparkSqlTestBase
import org.junit.jupiter.api.Assertions.assertEquals
@@ -120,11 +121,7 @@ class TestUpdateTable extends HoodieSparkSqlTestBase {
spark.sql(s"update $tableName set price = price * 2 where id = 1")
spark.sql(s"update $tableName set price = price * 2 where id = 1")
// verify compaction is complete
- val metaClient = HoodieTableMetaClient.builder()
- .setConf(spark.sparkContext.hadoopConfiguration)
- .setBasePath(tmp.getCanonicalPath + "/" + tableName)
- .build()
-
+ val metaClient = createMetaClient(spark, tmp.getCanonicalPath + "/" + tableName)
assertEquals(metaClient.getActiveTimeline.getLastCommitMetadataWithValidData.get.getLeft.getAction, "commit")
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestBootstrapProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestBootstrapProcedure.scala
index 90ed0906b1c..46de5b022bd 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestBootstrapProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestBootstrapProcedure.scala
@@ -18,10 +18,10 @@
package org.apache.spark.sql.hudi.procedure
import org.apache.hudi.common.model.HoodieTableType
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.functional.TestBootstrap
import org.apache.hudi.keygen.constant.KeyGeneratorOptions
import org.apache.hudi.storage.StoragePath
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.{Dataset, Row}
@@ -153,8 +153,7 @@ class TestBootstrapProcedure extends HoodieSparkProcedureTestBase {
result.length
}
- val metaClient = HoodieTableMetaClient.builder().setBasePath(tablePath)
- .setConf(spark.sessionState.newHadoopConf()).build()
+ val metaClient = createMetaClient(spark, tablePath)
assertResult("true") {
metaClient.getTableConfig.getString(KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
index 85829e378a6..e60a08fa197 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestClusteringProcedure.scala
@@ -22,10 +22,10 @@ package org.apache.spark.sql.hudi.procedure
import org.apache.hudi.DataSourceWriteOptions.{OPERATION, RECORDKEY_FIELD}
import org.apache.hudi.common.config.HoodieMetadataConfig
import org.apache.hudi.common.model.{HoodieCommitMetadata, WriteOperationType}
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.{HoodieActiveTimeline, HoodieInstant, HoodieTimeline}
-import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.util.collection.Pair
+import org.apache.hudi.common.util.{Option => HOption}
import org.apache.hudi.{DataSourceReadOptions, HoodieCLIUtils, HoodieDataSourceHelpers, HoodieFileIndex}
import org.apache.hadoop.conf.Configuration
@@ -35,6 +35,7 @@ import org.apache.spark.sql.types.{DataTypes, Metadata, StringType, StructField,
import org.apache.spark.sql.{Dataset, Row}
import java.util
+
import scala.collection.JavaConverters.asScalaIteratorConverter
class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
@@ -440,7 +441,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
spark.sql(s"call run_clustering(table => '$tableName', op => 'schedule')")
val conf = new Configuration
- val metaClient = HoodieTableMetaClient.builder.setConf(conf).setBasePath(basePath).build
+ val metaClient = HoodieTestUtils.createMetaClient(conf, basePath)
val instants = metaClient.getActiveTimeline.filterPendingReplaceTimeline().getInstants.iterator().asScala.map(_.getTimestamp).toSeq
assert(2 == instants.size)
@@ -504,7 +505,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
writeRecords(2, 4, 0, basePath, Map("hoodie.avro.schema.validate"-> "false"))
val conf = new Configuration
- val metaClient = HoodieTableMetaClient.builder.setConf(conf).setBasePath(basePath).build
+ val metaClient = HoodieTestUtils.createMetaClient(conf, basePath)
assert(0 == metaClient.getActiveTimeline.getCompletedReplaceTimeline.getInstants.size())
assert(metaClient.getActiveTimeline.filterPendingReplaceTimeline().empty())
@@ -575,7 +576,7 @@ class TestClusteringProcedure extends HoodieSparkProcedureTestBase {
// insert records
writeRecords(fileNum, numRecords, 0, basePath, metadataOpts ++ Map("hoodie.avro.schema.validate"-> "false"))
val conf = new Configuration
- val metaClient = HoodieTableMetaClient.builder.setConf(conf).setBasePath(basePath).build
+ val metaClient = HoodieTestUtils.createMetaClient(conf, basePath)
val avgSize = avgRecord(metaClient.getActiveTimeline)
val avgCount = Math.ceil(1.0 * numRecords / fileNum).toLong
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala
index fcbdc8df5d7..606fc8566a9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestCompactionProcedure.scala
@@ -19,8 +19,9 @@
package org.apache.spark.sql.hudi.procedure
-import org.apache.hudi.common.table.HoodieTableMetaClient
import org.apache.hudi.common.table.timeline.HoodieInstant
+import org.apache.hudi.common.testutils.HoodieTestUtils
+import org.apache.hudi.common.testutils.HoodieTestUtils.createMetaClient
import org.apache.hadoop.conf.Configuration
@@ -285,7 +286,7 @@ class TestCompactionProcedure extends HoodieSparkProcedureTestBase {
spark.sql(s"call run_compaction(table => '$tableName', op => 'schedule')")
- val metaClient = HoodieTableMetaClient.builder.setConf(new Configuration).setBasePath(tmp.getCanonicalPath).build
+ val metaClient = createMetaClient(tmp.getCanonicalPath)
val instants = metaClient.getActiveTimeline.filterPendingCompactionTimeline().getInstants
assertResult(1)(instants.size())
val ts = instants.get(0).getTimestamp
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
index 8588c1781ae..6316e8af9a5 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestRepairsProcedure.scala
@@ -24,20 +24,19 @@ import org.apache.hudi.common.table.timeline.HoodieTimeline
import org.apache.hudi.common.table.view.HoodieTableFileSystemView
import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, SchemaTestUtil}
import org.apache.hudi.hadoop.fs.HadoopFSUtils
-import org.apache.hudi.storage.{StoragePathInfo, StoragePath}
+import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import org.apache.hudi.testutils.HoodieSparkWriteableTestTable
import org.apache.avro.Schema
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
-import org.apache.spark.api.java.JavaSparkContext
import org.junit.jupiter.api.Assertions.assertEquals
import java.io.IOException
import java.net.URL
import java.nio.file.{Files, Paths}
import java.util.Properties
-
import scala.collection.JavaConverters.asScalaIteratorConverter
import scala.jdk.CollectionConverters.{asScalaSetConverter, iterableAsScalaIterableConverter}
@@ -65,10 +64,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
// create commit instant
Files.createFile(Paths.get(tablePath, ".hoodie", "100.commit"))
- val metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ val metaClient = createMetaClient(spark, tablePath)
// create partition path
val partition1 = Paths.get(tablePath, "2016/03/15").toString
@@ -169,7 +165,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
assertEquals(expectedOutput, actual)
spark.sql(s"""call repair_overwrite_hoodie_props(table => '$tableName', new_props_file_path => '${curPropPath}')""")
- val config = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(new Configuration()).build().getTableConfig
+ val config = createMetaClient(spark, tablePath).getTableConfig
val props = config.getProps
assertEquals(prevProps.size(), props.size())
props.entrySet().asScala.foreach((entry) => {
@@ -198,10 +194,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
| preCombineField = 'ts'
| )
""".stripMargin)
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
// Create four requested files
for (i <- 100 until 104) {
@@ -253,10 +246,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
| type = 'cow'
| )
""".stripMargin)
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
generateRecords(tablePath, bashPath, metaClient)
@@ -313,10 +303,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
| type = 'cow'
| )
""".stripMargin)
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
generateRecords(tablePath, bashPath, metaClient)
@@ -374,10 +361,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
| type = 'cow'
| )
""".stripMargin)
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
generateRecords(tablePath, bashPath, metaClient)
@@ -435,10 +419,7 @@ class TestRepairsProcedure extends HoodieSparkProcedureTestBase {
| type = 'cow'
| )
""".stripMargin)
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
generateRecords(tablePath, bashPath, metaClient)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
index 24f7deffcbe..e8289734afd 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
@@ -21,8 +21,7 @@ import org.apache.hudi.common.config.HoodieConfig
import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion}
import org.apache.hudi.common.util.{BinaryUtil, StringUtils}
import org.apache.hudi.storage.StoragePath
-
-import org.apache.spark.api.java.JavaSparkContext
+import org.apache.hudi.testutils.HoodieClientTestUtils.createMetaClient
import java.io.IOException
import java.time.Instant
@@ -52,10 +51,7 @@ class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase {
checkExceptionContain(s"""call downgrade_table(table => '$tableName')""")(
s"Argument: to_version is required")
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
// verify hoodie.table.version of the original table
assertResult(HoodieTableVersion.SIX.versionCode) {
@@ -107,10 +103,7 @@ class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase {
// downgrade table to THREE
checkAnswer(s"""call downgrade_table(table => '$tableName', to_version => 'THREE')""")(Seq(true))
- var metaClient = HoodieTableMetaClient.builder
- .setConf(new JavaSparkContext(spark.sparkContext).hadoopConfiguration())
- .setBasePath(tablePath)
- .build
+ var metaClient = createMetaClient(spark, tablePath)
val storage = metaClient.getStorage
// verify hoodie.table.version of the table is THREE
assertResult(HoodieTableVersion.THREE.versionCode) {
diff --git a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index a755c5ba4f2..f2c67bc22e5 100644
--- a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++ b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieInstantTimeGenerator;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.testutils.NetworkTestUtils;
import org.apache.hudi.common.testutils.SchemaTestUtil;
import org.apache.hudi.common.util.ConfigUtils;
@@ -448,8 +449,8 @@ public class TestHiveSyncTool {
HiveTestUtil.removeCommitFromActiveTimeline("300", COMMIT_ACTION);
HiveTestUtil.removeCommitFromActiveTimeline("500", COMMIT_ACTION);
HiveTestUtil.removeCommitFromActiveTimeline("600", COMMIT_ACTION);
- HoodieTableMetaClient metaClient = HoodieTableMetaClient.builder()
- .setConf(hiveClient.config.getHadoopConf()).setBasePath(basePath).build();
+ HoodieTableMetaClient metaClient = HoodieTestUtils.createMetaClient(
+ hiveClient.config.getHadoopConf(), basePath);
assertEquals(
Arrays.asList("400", "700", "800"),
metaClient.getActiveTimeline().getInstants().stream()