You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2022/01/19 23:08:26 UTC
[hudi] 05/06: Removing a extraneous test class
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.10.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 5dada430ec3b2e34cd64b2f589163cebcf51c469
Author: sivabalan <n....@gmail.com>
AuthorDate: Wed Jan 19 18:02:07 2022 -0500
Removing a extraneous test class
---
.../TestHoodieSparkMergeOnReadTableCompaction.java | 125 ---------------------
1 file changed, 125 deletions(-)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java
deleted file mode 100644
index 13903bf..0000000
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/functional/TestHoodieSparkMergeOnReadTableCompaction.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hudi.table.functional;
-
-import org.apache.hudi.client.SparkRDDWriteClient;
-import org.apache.hudi.client.WriteStatus;
-import org.apache.hudi.common.model.HoodieTableType;
-import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
-import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
-import org.apache.hudi.common.util.Option;
-import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieIndexConfig;
-import org.apache.hudi.config.HoodieLayoutConfig;
-import org.apache.hudi.config.HoodieStorageConfig;
-import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.index.HoodieIndex;
-import org.apache.hudi.table.action.commit.SparkBucketIndexPartitioner;
-import org.apache.hudi.table.storage.HoodieStorageLayout;
-import org.apache.hudi.testutils.HoodieMergeOnReadTestUtils;
-import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
-
-import org.apache.spark.api.java.JavaRDD;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.api.Test;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
-import static org.apache.hudi.config.HoodieWriteConfig.AUTO_COMMIT_ENABLE;
-
-@Tag("functional")
-public class TestHoodieSparkMergeOnReadTableCompaction extends SparkClientFunctionalTestHarness {
-
- private HoodieTestDataGenerator dataGen;
- private SparkRDDWriteClient client;
- private HoodieTableMetaClient metaClient;
-
- @BeforeEach
- public void setup() {
- dataGen = new HoodieTestDataGenerator();
- }
-
- @Test
- public void testWriteDuringCompaction() throws IOException {
- HoodieWriteConfig config = HoodieWriteConfig.newBuilder()
- .forTable("test-trip-table")
- .withPath(basePath())
- .withSchema(TRIP_EXAMPLE_SCHEMA)
- .withParallelism(2, 2)
- .withAutoCommit(false)
- .withCompactionConfig(HoodieCompactionConfig.newBuilder()
- .withMaxNumDeltaCommitsBeforeCompaction(1).build())
- .withStorageConfig(HoodieStorageConfig.newBuilder()
- .parquetMaxFileSize(1024).build())
- .withLayoutConfig(HoodieLayoutConfig.newBuilder()
- .withLayoutType(HoodieStorageLayout.LayoutType.BUCKET.name())
- .withLayoutPartitioner(SparkBucketIndexPartitioner.class.getName()).build())
- .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(HoodieIndex.IndexType.BUCKET).withBucketNum("1").build()).build();
- metaClient = getHoodieMetaClient(HoodieTableType.MERGE_ON_READ, config.getProps());
- client = getHoodieWriteClient(config);
-
- // write data and commit
- writeData(HoodieActiveTimeline.createNewInstantTime(), 100, true);
- // write data again, and in the case of bucket index, all records will go into log files (we use a small max_file_size)
- writeData(HoodieActiveTimeline.createNewInstantTime(), 100, true);
- Assertions.assertEquals(200, readTableTotalRecordsNum());
- // schedule compaction
- String compactionTime = (String) client.scheduleCompaction(Option.empty()).get();
- // write data, and do not commit. those records should not visible to reader
- String insertTime = HoodieActiveTimeline.createNewInstantTime();
- List<WriteStatus> writeStatuses = writeData(insertTime, 100, false);
- Assertions.assertEquals(200, readTableTotalRecordsNum());
- // commit the write. The records should be visible now even though the compaction does not complete.
- client.commitStats(insertTime, writeStatuses.stream().map(WriteStatus::getStat).collect(Collectors.toList()), Option.empty(), metaClient.getCommitActionType());
- Assertions.assertEquals(300, readTableTotalRecordsNum());
- // after the compaction, total records should remain the same
- config.setValue(AUTO_COMMIT_ENABLE, "true");
- client.compact(compactionTime);
- Assertions.assertEquals(300, readTableTotalRecordsNum());
- }
-
- private long readTableTotalRecordsNum() {
- return HoodieMergeOnReadTestUtils.getRecordsUsingInputFormat(hadoopConf(),
- Arrays.stream(dataGen.getPartitionPaths()).map(p -> Paths.get(basePath(), p).toString()).collect(Collectors.toList()), basePath()).size();
- }
-
- private List<WriteStatus> writeData(String instant, int numRecords, boolean doCommit) {
- metaClient = HoodieTableMetaClient.reload(metaClient);
- JavaRDD records = jsc().parallelize(dataGen.generateInserts(instant, numRecords), 2);
- metaClient = HoodieTableMetaClient.reload(metaClient);
- client.startCommitWithTime(instant);
- List<WriteStatus> writeStatues = client.upsert(records, instant).collect();
- org.apache.hudi.testutils.Assertions.assertNoWriteErrors(writeStatues);
- if (doCommit) {
- Assertions.assertTrue(client.commitStats(instant, writeStatues.stream().map(WriteStatus::getStat).collect(Collectors.toList()),
- Option.empty(), metaClient.getCommitActionType()));
- }
- metaClient = HoodieTableMetaClient.reload(metaClient);
- return writeStatues;
- }
-}
\ No newline at end of file