You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2020/10/12 06:39:21 UTC
[hudi] branch master updated: [HUDI-995] Migrate HoodieTestUtils
APIs to HoodieTestTable (#2167)
This is an automated email from the ASF dual-hosted git repository.
vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c5e10d6 [HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2167)
c5e10d6 is described below
commit c5e10d668f9366f29bdf7721f7efe4140782527b
Author: Raymond Xu <27...@users.noreply.github.com>
AuthorDate: Sun Oct 11 23:39:10 2020 -0700
[HUDI-995] Migrate HoodieTestUtils APIs to HoodieTestTable (#2167)
Remove APIs in `HoodieTestUtils`
- `createCommitFiles`
- `createDataFile`
- `createNewLogFile`
- `createCompactionRequest`
Migrated usages in `TestCleaner#testPendingCompactions`.
Also improved some API names in `HoodieTestTable`.
---
.../hudi/cli/integ/ITTestRepairsCommand.java | 2 +-
.../java/org/apache/hudi/table/TestCleaner.java | 150 +++++++++++----------
.../rollback/TestMarkerBasedRollbackStrategy.java | 8 +-
.../hudi/common/testutils/HoodieTestTable.java | 31 ++++-
.../hudi/common/testutils/HoodieTestUtils.java | 75 -----------
.../hudi/hadoop/TestHoodieROTablePathFilter.java | 2 +-
6 files changed, 112 insertions(+), 156 deletions(-)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
index f277e33..133dcb0 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/integ/ITTestRepairsCommand.java
@@ -87,7 +87,7 @@ public class ITTestRepairsCommand extends AbstractShellIntegrationTest {
testTable.addCommit("20160401010101")
.withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "1", hoodieRecords1)
.withInserts(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, "2", hoodieRecords2)
- .withLogFile(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH);
+ .getFileIdWithLogFile(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH);
testTable.withInserts(HoodieTestDataGenerator.DEFAULT_SECOND_PARTITION_PATH, "4", hoodieRecords1)
.withInserts(HoodieTestDataGenerator.DEFAULT_THIRD_PARTITION_PATH, "6", hoodieRecords1);
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
index 152a981..00f1ea0 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/TestCleaner.java
@@ -51,7 +51,6 @@ import org.apache.hudi.common.table.timeline.versioning.clean.CleanMetadataMigra
import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanMigrator;
import org.apache.hudi.common.table.timeline.versioning.clean.CleanPlanV1MigrationHandler;
import org.apache.hudi.common.table.view.TableFileSystemView;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestTable;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.CleanerUtils;
@@ -155,7 +154,7 @@ public class TestCleaner extends HoodieClientTestBase {
assertTrue(table.getCompletedCleanTimeline().empty());
HoodieIndex index = SparkHoodieIndex.createIndex(cfg);
- List<HoodieRecord> taggedRecords = ((JavaRDD<HoodieRecord>)index.tagLocation(jsc.parallelize(records, 1), context, table)).collect();
+ List<HoodieRecord> taggedRecords = ((JavaRDD<HoodieRecord>) index.tagLocation(jsc.parallelize(records, 1), context, table)).collect();
checkTaggedRecords(taggedRecords, newCommitTime);
}
@@ -550,7 +549,7 @@ public class TestCleaner extends HoodieClientTestBase {
Map<String, String> partitionAndFileId002 = testTable.addCommit("00000000000002")
.withBaseFilesInPartition(p0, file1P0C0)
.withBaseFilesInPartition(p1, file1P1C0)
- .withBaseFilesInPartitions(p0, p1);
+ .getFileIdsWithBaseFilesInPartitions(p0, p1);
List<HoodieCleanStat> hoodieCleanStatsTwo = runCleaner(config, 1);
// enableBootstrapSourceClean would delete the bootstrap base file as the same time
@@ -592,7 +591,7 @@ public class TestCleaner extends HoodieClientTestBase {
// make next commit, with 2 updates to existing files, and 1 insert
String file3P0C2 = testTable.addCommit("00000000000003")
.withBaseFilesInPartition(p0, file1P0C0, file2P0C1)
- .withBaseFilesInPartitions(p0).get(p0);
+ .getFileIdsWithBaseFilesInPartitions(p0).get(p0);
List<HoodieCleanStat> hoodieCleanStatsThree = runCleaner(config, 3);
assertEquals(2,
getCleanStat(hoodieCleanStatsThree, p0)
@@ -625,7 +624,7 @@ public class TestCleaner extends HoodieClientTestBase {
String p0 = "2020/01/01";
// Make 3 files, one base file and 2 log files associated with base file
- String file1P0 = testTable.addDeltaCommit("000").withBaseFilesInPartitions(p0).get(p0);
+ String file1P0 = testTable.addDeltaCommit("000").getFileIdsWithBaseFilesInPartitions(p0).get(p0);
testTable.forDeltaCommit("000")
.withLogFile(p0, file1P0, 1)
.withLogFile(p0, file1P0, 2);
@@ -865,7 +864,7 @@ public class TestCleaner extends HoodieClientTestBase {
assertTrue(testTable.baseFileExists(p1, "00000000000001", file1P1C0));
// make next commit, with 1 insert & 1 update per partition
- Map<String, String> partitionAndFileId002 = testTable.addInflightCommit("00000000000002").withBaseFilesInPartitions(p0, p1);
+ Map<String, String> partitionAndFileId002 = testTable.addInflightCommit("00000000000002").getFileIdsWithBaseFilesInPartitions(p0, p1);
String file2P0C1 = partitionAndFileId002.get(p0);
String file2P1C1 = partitionAndFileId002.get(p1);
testTable.forCommit("00000000000002").withBaseFilesInPartition(p0, file1P0C0).withBaseFilesInPartition(p1, file1P1C0);
@@ -889,7 +888,7 @@ public class TestCleaner extends HoodieClientTestBase {
String file3P0C2 = testTable.addInflightCommit("00000000000003")
.withBaseFilesInPartition(p0, file1P0C0)
.withBaseFilesInPartition(p0, file2P0C1)
- .withBaseFilesInPartitions(p0).get(p0);
+ .getFileIdsWithBaseFilesInPartitions(p0).get(p0);
commitMetadata = generateCommitMetadata(CollectionUtils
.createImmutableMap(p0,
CollectionUtils.createImmutableList(file1P0C0, file2P0C1, file3P0C2)));
@@ -906,7 +905,7 @@ public class TestCleaner extends HoodieClientTestBase {
String file4P0C3 = testTable.addInflightCommit("00000000000004")
.withBaseFilesInPartition(p0, file1P0C0)
.withBaseFilesInPartition(p0, file2P0C1)
- .withBaseFilesInPartitions(p0).get(p0);
+ .getFileIdsWithBaseFilesInPartitions(p0).get(p0);
commitMetadata = generateCommitMetadata(CollectionUtils.createImmutableMap(
p0, CollectionUtils.createImmutableList(file1P0C0, file2P0C1, file4P0C3)));
metaClient.getActiveTimeline().saveAsComplete(
@@ -1021,7 +1020,7 @@ public class TestCleaner extends HoodieClientTestBase {
* Test Keep Latest Commits when there are pending compactions.
*/
@Test
- public void testKeepLatestCommitsWithPendingCompactions() throws IOException {
+ public void testKeepLatestCommitsWithPendingCompactions() throws Exception {
HoodieWriteConfig config = HoodieWriteConfig.newBuilder().withPath(basePath).withAssumeDatePartitioning(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
.withCleanerPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS).retainCommits(2).build())
@@ -1043,7 +1042,7 @@ public class TestCleaner extends HoodieClientTestBase {
*/
@ParameterizedTest
@ValueSource(booleans = {false, true})
- public void testKeepLatestVersionsWithPendingCompactions(boolean retryFailure) throws IOException {
+ public void testKeepLatestVersionsWithPendingCompactions(boolean retryFailure) throws Exception {
HoodieWriteConfig config =
HoodieWriteConfig.newBuilder().withPath(basePath).withAssumeDatePartitioning(true)
.withCompactionConfig(HoodieCompactionConfig.newBuilder()
@@ -1098,73 +1097,82 @@ public class TestCleaner extends HoodieClientTestBase {
* @param expNumFilesDeleted Number of files deleted
*/
private void testPendingCompactions(HoodieWriteConfig config, int expNumFilesDeleted,
- int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws IOException {
+ int expNumFilesUnderCompactionDeleted, boolean retryFailure) throws Exception {
HoodieTableMetaClient metaClient =
HoodieTestUtils.init(hadoopConf, basePath, HoodieTableType.MERGE_ON_READ);
- String[] instants = new String[] {"000", "001", "003", "005", "007", "009", "011", "013"};
- String[] compactionInstants = new String[] {"002", "004", "006", "008", "010"};
- Map<String, String> expFileIdToPendingCompaction = new HashMap<>();
- Map<String, String> fileIdToLatestInstantBeforeCompaction = new HashMap<>();
- Map<String, List<FileSlice>> compactionInstantsToFileSlices = new HashMap<>();
-
- for (String instant : instants) {
- HoodieTestUtils.createCommitFiles(basePath, instant);
- }
+ final String partition = "2016/03/15";
+ Map<String, String> expFileIdToPendingCompaction = new HashMap<String, String>() {
+ {
+ put("fileId2", "004");
+ put("fileId3", "006");
+ put("fileId4", "008");
+ put("fileId5", "010");
+ }
+ };
+ Map<String, String> fileIdToLatestInstantBeforeCompaction = new HashMap<String, String>() {
+ {
+ put("fileId1", "000");
+ put("fileId2", "000");
+ put("fileId3", "001");
+ put("fileId4", "003");
+ put("fileId5", "005");
+ put("fileId6", "009");
+ put("fileId7", "011");
+ }
+ };
// Generate 7 file-groups. First one has only one slice and no pending compaction. File Slices (2 - 5) has
// multiple versions with pending compaction. File Slices (6 - 7) have multiple file-slices but not under
// compactions
// FileIds 2-5 will be under compaction
- int maxNumFileIds = 7;
- String[] fileIds = new String[] {"fileId1", "fileId2", "fileId3", "fileId4", "fileId5", "fileId6", "fileId7"};
- int maxNumFileIdsForCompaction = 4;
- for (int i = 0; i < maxNumFileIds; i++) {
- final String fileId = HoodieTestUtils.createDataFile(basePath,
- HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0], fileIds[i]);
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0],
- fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[0],
- fileId, Option.of(2));
- fileIdToLatestInstantBeforeCompaction.put(fileId, instants[0]);
- for (int j = 1; j <= i; j++) {
- if (j == i && j <= maxNumFileIdsForCompaction) {
- expFileIdToPendingCompaction.put(fileId, compactionInstants[j]);
- metaClient = HoodieTableMetaClient.reload(metaClient);
- HoodieTable table = HoodieSparkTable.create(config, context, metaClient);
- FileSlice slice =
- table.getSliceView().getLatestFileSlices(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH)
- .filter(fs -> fs.getFileId().equals(fileId)).findFirst().get();
- List<FileSlice> slices = new ArrayList<>();
- if (compactionInstantsToFileSlices.containsKey(compactionInstants[j])) {
- slices = compactionInstantsToFileSlices.get(compactionInstants[j]);
- }
- slices.add(slice);
- compactionInstantsToFileSlices.put(compactionInstants[j], slices);
- // Add log-files to simulate delta-commits after pending compaction
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- compactionInstants[j], fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- compactionInstants[j], fileId, Option.of(2));
- } else {
- HoodieTestUtils.createDataFile(basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, instants[j],
- fileId);
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- instants[j], fileId, Option.empty());
- HoodieTestUtils.createNewLogFile(fs, basePath, HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH,
- instants[j], fileId, Option.of(2));
- fileIdToLatestInstantBeforeCompaction.put(fileId, instants[j]);
- }
- }
- }
-
- // Setup pending compaction plans
- for (String instant : compactionInstants) {
- List<FileSlice> fileSliceList = compactionInstantsToFileSlices.get(instant);
- if (null != fileSliceList) {
- HoodieTestUtils.createCompactionRequest(metaClient, instant, fileSliceList.stream()
- .map(fs -> Pair.of(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, fs)).collect(Collectors.toList()));
- }
- }
+ HoodieTestTable.of(metaClient)
+ .addCommit("000")
+ .withBaseFilesInPartition(partition, "fileId1", "fileId2", "fileId3", "fileId4", "fileId5", "fileId6", "fileId7")
+ .withLogFile(partition, "fileId1", 1, 2)
+ .withLogFile(partition, "fileId2", 1, 2)
+ .withLogFile(partition, "fileId3", 1, 2)
+ .withLogFile(partition, "fileId4", 1, 2)
+ .withLogFile(partition, "fileId5", 1, 2)
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addCommit("001")
+ .withBaseFilesInPartition(partition, "fileId3", "fileId4", "fileId5", "fileId6", "fileId7")
+ .withLogFile(partition, "fileId3", 1, 2)
+ .withLogFile(partition, "fileId4", 1, 2)
+ .withLogFile(partition, "fileId5", 1, 2)
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addCommit("003")
+ .withBaseFilesInPartition(partition, "fileId4", "fileId5", "fileId6", "fileId7")
+ .withLogFile(partition, "fileId4", 1, 2)
+ .withLogFile(partition, "fileId5", 1, 2)
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addRequestedCompaction("004", new FileSlice(partition, "000", "fileId2"))
+ .withLogFile(partition, "fileId2", 1, 2)
+ .addCommit("005")
+ .withBaseFilesInPartition(partition, "fileId5", "fileId6", "fileId7")
+ .withLogFile(partition, "fileId5", 1, 2)
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addRequestedCompaction("006", new FileSlice(partition, "001", "fileId3"))
+ .withLogFile(partition, "fileId3", 1, 2)
+ .addCommit("007")
+ .withBaseFilesInPartition(partition, "fileId6", "fileId7")
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addRequestedCompaction("008", new FileSlice(partition, "003", "fileId4"))
+ .withLogFile(partition, "fileId4", 1, 2)
+ .addCommit("009")
+ .withBaseFilesInPartition(partition, "fileId6", "fileId7")
+ .withLogFile(partition, "fileId6", 1, 2)
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addRequestedCompaction("010", new FileSlice(partition, "005", "fileId5"))
+ .withLogFile(partition, "fileId5", 1, 2)
+ .addCommit("011")
+ .withBaseFilesInPartition(partition, "fileId7")
+ .withLogFile(partition, "fileId7", 1, 2)
+ .addCommit("013");
// Clean now
metaClient = HoodieTableMetaClient.reload(metaClient);
@@ -1177,7 +1185,7 @@ public class TestCleaner extends HoodieClientTestBase {
expFileIdToPendingCompaction.forEach((fileId, value) -> {
String baseInstantForCompaction = fileIdToLatestInstantBeforeCompaction.get(fileId);
Option<FileSlice> fileSliceForCompaction = Option.fromJavaOptional(hoodieTable.getSliceView()
- .getLatestFileSlicesBeforeOrOn(HoodieTestDataGenerator.DEFAULT_FIRST_PARTITION_PATH, baseInstantForCompaction,
+ .getLatestFileSlicesBeforeOrOn(partition, baseInstantForCompaction,
true)
.filter(fs -> fs.getFileId().equals(fileId)).findFirst());
assertTrue(fileSliceForCompaction.isPresent(), "Base Instant for Compaction must be preserved");
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java
index 191e90f..7acff79 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/rollback/TestMarkerBasedRollbackStrategy.java
@@ -59,10 +59,10 @@ public class TestMarkerBasedRollbackStrategy extends HoodieClientTestBase {
// given: wrote some base files and corresponding markers
HoodieTestTable testTable = HoodieTestTable.of(metaClient);
String f0 = testTable.addRequestedCommit("000")
- .withBaseFilesInPartitions("partA").get("partA");
+ .getFileIdsWithBaseFilesInPartitions("partA").get("partA");
String f1 = testTable.addCommit("001")
.withBaseFilesInPartition("partA", f0)
- .withBaseFilesInPartitions("partB").get("partB");
+ .getFileIdsWithBaseFilesInPartitions("partB").get("partB");
String f2 = "f2";
testTable.forCommit("001")
.withMarkerFile("partA", f0, IOType.MERGE)
@@ -90,10 +90,10 @@ public class TestMarkerBasedRollbackStrategy extends HoodieClientTestBase {
// given: wrote some base + log files and corresponding markers
HoodieTestTable testTable = HoodieTestTable.of(metaClient);
String f2 = testTable.addRequestedDeltaCommit("000")
- .withBaseFilesInPartitions("partA").get("partA");
+ .getFileIdsWithBaseFilesInPartitions("partA").get("partA");
String f1 = testTable.addDeltaCommit("001")
.withLogFile("partA", f2)
- .withBaseFilesInPartitions("partB").get("partB");
+ .getFileIdsWithBaseFilesInPartitions("partB").get("partB");
String f3 = "f3";
String f4 = "f4";
testTable.forDeltaCommit("001")
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
index 9cacf1f..3663917 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestTable.java
@@ -21,12 +21,19 @@ package org.apache.hudi.common.testutils;
import org.apache.hudi.avro.model.HoodieCleanMetadata;
import org.apache.hudi.avro.model.HoodieCleanerPlan;
+import org.apache.hudi.avro.model.HoodieCompactionPlan;
+import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieReplaceCommitMetadata;
import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.timeline.HoodieInstant;
import org.apache.hudi.common.table.timeline.HoodieTimeline;
+import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
+import org.apache.hudi.common.util.CompactionUtils;
+import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ValidationUtils;
+import org.apache.hudi.common.util.collection.Pair;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -189,6 +196,20 @@ public class HoodieTestTable {
return this;
}
+ public HoodieTestTable addRequestedCompaction(String instantTime, HoodieCompactionPlan compactionPlan) throws IOException {
+ HoodieInstant compactionInstant = new HoodieInstant(HoodieInstant.State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instantTime);
+ metaClient.getActiveTimeline().saveToCompactionRequested(compactionInstant,
+ TimelineMetadataUtils.serializeCompactionPlan(compactionPlan));
+ return addRequestedCompaction(instantTime);
+ }
+
+ public HoodieTestTable addRequestedCompaction(String instantTime, FileSlice... fileSlices) throws IOException {
+ HoodieCompactionPlan plan = CompactionUtils
+ .buildFromFileSlices(Arrays.stream(fileSlices).map(fs -> Pair.of(fs.getPartitionPath(), fs))
+ .collect(Collectors.toList()), Option.empty(), Option.empty());
+ return addRequestedCompaction(instantTime, plan);
+ }
+
public HoodieTestTable addCompaction(String instantTime) throws IOException {
createRequestedCompaction(basePath, instantTime);
createInflightCompaction(basePath, instantTime);
@@ -245,7 +266,7 @@ public class HoodieTestTable {
*
* @return A {@link Map} of partition and its newly inserted file's id.
*/
- public Map<String, String> withBaseFilesInPartitions(String... partitions) throws Exception {
+ public Map<String, String> getFileIdsWithBaseFilesInPartitions(String... partitions) throws Exception {
Map<String, String> partitionFileIdMap = new HashMap<>();
for (String p : partitions) {
String fileId = UUID.randomUUID().toString();
@@ -277,7 +298,7 @@ public class HoodieTestTable {
return this;
}
- public String withLogFile(String partitionPath) throws Exception {
+ public String getFileIdWithLogFile(String partitionPath) throws Exception {
String fileId = UUID.randomUUID().toString();
withLogFile(partitionPath, fileId);
return fileId;
@@ -287,8 +308,10 @@ public class HoodieTestTable {
return withLogFile(partitionPath, fileId, 0);
}
- public HoodieTestTable withLogFile(String partitionPath, String fileId, int version) throws Exception {
- FileCreateUtils.createLogFile(basePath, partitionPath, currentInstantTime, fileId, version);
+ public HoodieTestTable withLogFile(String partitionPath, String fileId, int... versions) throws Exception {
+ for (int version : versions) {
+ FileCreateUtils.createLogFile(basePath, partitionPath, currentInstantTime, fileId, version);
+ }
return this;
}
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index 1f86a59..d94f41f 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -18,9 +18,6 @@
package org.apache.hudi.common.testutils;
-import org.apache.hudi.avro.model.HoodieCompactionPlan;
-import org.apache.hudi.common.fs.FSUtils;
-import org.apache.hudi.common.model.FileSlice;
import org.apache.hudi.common.model.HoodieAvroPayload;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.model.HoodieTableType;
@@ -28,27 +25,15 @@ import org.apache.hudi.common.model.HoodieWriteStat;
import org.apache.hudi.common.model.HoodieWriteStat.RuntimeStats;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.log.HoodieLogFormat;
-import org.apache.hudi.common.table.timeline.HoodieInstant;
-import org.apache.hudi.common.table.timeline.HoodieInstant.State;
-import org.apache.hudi.common.table.timeline.HoodieTimeline;
-import org.apache.hudi.common.table.timeline.TimelineMetadataUtils;
-import org.apache.hudi.common.util.CompactionUtils;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.common.util.collection.Pair;
import com.esotericsoftware.kryo.Kryo;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;
import com.esotericsoftware.kryo.serializers.JavaSerializer;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.StringUtils;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
@@ -122,66 +107,6 @@ public class HoodieTestUtils {
return HoodieTableMetaClient.initTableAndGetMetaClient(hadoopConf, basePath, properties);
}
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static void createCommitFiles(String basePath, String... instantTimes) throws IOException {
- for (String instantTime : instantTimes) {
- new File(
- basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
- + HoodieTimeline.makeRequestedCommitFileName(instantTime)).createNewFile();
- new File(
- basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/"
- + HoodieTimeline.makeInflightCommitFileName(instantTime)).createNewFile();
- new File(
- basePath + "/" + HoodieTableMetaClient.METAFOLDER_NAME + "/" + HoodieTimeline.makeCommitFileName(instantTime))
- .createNewFile();
- }
- }
-
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static String createDataFile(String basePath, String partitionPath, String instantTime, String fileID)
- throws IOException {
- String folderPath = basePath + "/" + partitionPath + "/";
- new File(folderPath).mkdirs();
- new File(folderPath + FSUtils.makeDataFileName(instantTime, DEFAULT_WRITE_TOKEN, fileID)).createNewFile();
- return fileID;
- }
-
- /**
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static String createNewLogFile(FileSystem fs, String basePath, String partitionPath, String instantTime,
- String fileID, Option<Integer> version) throws IOException {
- String folderPath = basePath + "/" + partitionPath + "/";
- boolean makeDir = fs.mkdirs(new Path(folderPath));
- if (!makeDir) {
- throw new IOException("cannot create directory for path " + folderPath);
- }
- boolean createFile = fs.createNewFile(new Path(folderPath + FSUtils.makeLogFileName(fileID, ".log", instantTime,
- version.orElse(DEFAULT_LOG_VERSION), HoodieLogFormat.UNKNOWN_WRITE_TOKEN)));
- if (!createFile) {
- throw new IOException(
- StringUtils.format("cannot create data file for commit %s and fileId %s", instantTime, fileID));
- }
- return fileID;
- }
-
- /**
- * TODO: incorporate into {@link HoodieTestTable}.
- *
- * @deprecated Use {@link HoodieTestTable} instead.
- */
- public static void createCompactionRequest(HoodieTableMetaClient metaClient, String instant,
- List<Pair<String, FileSlice>> fileSliceList) throws IOException {
- HoodieCompactionPlan plan = CompactionUtils.buildFromFileSlices(fileSliceList, Option.empty(), Option.empty());
- HoodieInstant compactionInstant = new HoodieInstant(State.REQUESTED, HoodieTimeline.COMPACTION_ACTION, instant);
- metaClient.getActiveTimeline().saveToCompactionRequested(compactionInstant,
- TimelineMetadataUtils.serializeCompactionPlan(plan));
- }
-
public static <T extends Serializable> T serializeDeserialize(T object, Class<T> clazz) {
// Using Kyro as the default serializer in Spark Jobs
Kryo kryo = new Kryo();
diff --git a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
index 5a6070f..ba88df3 100644
--- a/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
+++ b/hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/TestHoodieROTablePathFilter.java
@@ -88,7 +88,7 @@ public class TestHoodieROTablePathFilter extends HoodieCommonTestHarness {
public void testPartitionPathsAsNonHoodiePaths() throws Exception {
final String p1 = "2017/01/01";
final String p2 = "2017/01/02";
- testTable.addCommit("001").withBaseFilesInPartitions(p1, p2);
+ testTable.addCommit("001").getFileIdsWithBaseFilesInPartitions(p1, p2);
Path partitionPath1 = testTable.getPartitionPath(p1).getParent();
Path partitionPath2 = testTable.getPartitionPath(p2).getParent();
assertTrue(pathFilter.accept(partitionPath1), "Directories should be accepted");