You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2020/07/21 17:23:55 UTC
[hudi] branch master updated: [HUDI-994] Move TestHoodieIndex test
cases to unit tests (#1850)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 5e7ab11 [HUDI-994] Move TestHoodieIndex test cases to unit tests (#1850)
5e7ab11 is described below
commit 5e7ab11e2ead23428ed5089421f2abd6433fe8e5
Author: Raymond Xu <27...@users.noreply.github.com>
AuthorDate: Tue Jul 21 10:23:43 2020 -0700
[HUDI-994] Move TestHoodieIndex test cases to unit tests (#1850)
---
.../org/apache/hudi/index/TestHoodieIndex.java | 155 +-----------------
.../apache/hudi/index/TestHoodieIndexConfigs.java | 181 +++++++++++++++++++++
2 files changed, 187 insertions(+), 149 deletions(-)
diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java
index 66b8ae6..ab3e888 100644
--- a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java
+++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndex.java
@@ -26,7 +26,6 @@ import org.apache.hudi.common.model.EmptyHoodieRecordPayload;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodiePartitionMetadata;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.common.model.HoodieRecordPayload;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.view.FileSystemViewStorageConfig;
import org.apache.hudi.common.table.view.FileSystemViewStorageType;
@@ -34,17 +33,10 @@ import org.apache.hudi.common.util.FileIOUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.config.HoodieCompactionConfig;
-import org.apache.hudi.config.HoodieHBaseIndexConfig;
import org.apache.hudi.config.HoodieIndexConfig;
import org.apache.hudi.config.HoodieStorageConfig;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieException;
-import org.apache.hudi.exception.HoodieIndexException;
import org.apache.hudi.index.HoodieIndex.IndexType;
-import org.apache.hudi.index.bloom.HoodieBloomIndex;
-import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex;
-import org.apache.hudi.index.hbase.HBaseIndex;
-import org.apache.hudi.index.simple.HoodieSimpleIndex;
import org.apache.hudi.table.HoodieTable;
import org.apache.hudi.testutils.HoodieClientTestHarness;
import org.apache.hudi.testutils.HoodieClientTestUtils;
@@ -55,9 +47,7 @@ import org.apache.avro.Schema;
import org.apache.hadoop.fs.Path;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
-import org.apache.spark.api.java.JavaSparkContext;
import org.junit.jupiter.api.AfterEach;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.EnumSource;
@@ -76,7 +66,6 @@ import scala.Tuple2;
import static org.apache.hudi.testutils.Assertions.assertNoWriteErrors;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
-import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieIndex extends HoodieClientTestHarness {
@@ -85,22 +74,19 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
private IndexType indexType;
private HoodieIndex index;
private HoodieWriteConfig config;
- private String schemaStr;
private Schema schema;
private void setUp(IndexType indexType) throws Exception {
- setUp(indexType, true);
- }
-
- private void setUp(IndexType indexType, boolean initializeIndex) throws Exception {
this.indexType = indexType;
initResources();
// We have some records to be tagged (two different partitions)
- schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt"));
+ String schemaStr = FileIOUtils.readAsUTFString(getClass().getResourceAsStream("/exampleSchema.txt"));
schema = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(schemaStr));
- if (initializeIndex) {
- instantiateIndex();
- }
+ config = getConfigBuilder()
+ .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType)
+ .build()).withAutoCommit(false).build();
+ writeClient = getHoodieWriteClient(config);
+ this.index = writeClient.getIndex();
}
@AfterEach
@@ -109,75 +95,6 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
}
@ParameterizedTest
- @EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE", "HBASE"})
- public void testCreateIndex(IndexType indexType) throws Exception {
- setUp(indexType, false);
- HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
- HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
- switch (indexType) {
- case INMEMORY:
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
- assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex);
- break;
- case BLOOM:
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
- assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex);
- break;
- case GLOBAL_BLOOM:
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build();
- assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex);
- break;
- case SIMPLE:
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.SIMPLE).build()).build();
- assertTrue(HoodieIndex.createIndex(config) instanceof HoodieSimpleIndex);
- break;
- case HBASE:
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE)
- .withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build())
- .build();
- assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex);
- break;
- default:
- // no -op. just for checkstyle errors
- }
- }
-
- @Test
- public void testCreateDummyIndex() throws Exception {
- setUp(IndexType.BLOOM, false);
- HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
- HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
- config = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build();
- assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex);
- }
-
- @Test
- public void testCreateIndex_withException() throws Exception {
- setUp(IndexType.BLOOM, false);
- HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
- HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
- final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build();
- final Throwable thrown1 = assertThrows(HoodieException.class, () -> {
- HoodieIndex.createIndex(config1);
- }, "exception is expected");
- assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex"));
-
- final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath)
- .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build();
- final Throwable thrown2 = assertThrows(HoodieException.class, () -> {
- HoodieIndex.createIndex(config2);
- }, "exception is expected");
- assertTrue(thrown2.getMessage().contains("Unable to instantiate class"));
- }
-
- @ParameterizedTest
@EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE"})
public void testSimpleTagLocationAndUpdate(IndexType indexType) throws Exception {
setUp(indexType);
@@ -537,64 +454,4 @@ public class TestHoodieIndex extends HoodieClientTestHarness {
.withStorageType(FileSystemViewStorageType.EMBEDDED_KV_STORE).build());
}
- private void instantiateIndex() {
- config = getConfigBuilder()
- .withIndexConfig(HoodieIndexConfig.newBuilder().withIndexType(indexType)
- .build()).withAutoCommit(false).build();
- writeClient = getHoodieWriteClient(config);
- this.index = writeClient.getIndex();
- }
-
- public static class DummyHoodieIndex<T extends HoodieRecordPayload> extends HoodieIndex<T> {
-
- public DummyHoodieIndex(HoodieWriteConfig config) {
- super(config);
- }
-
- @Override
- public JavaPairRDD<HoodieKey, Option<Pair<String, String>>> fetchRecordLocation(JavaRDD<HoodieKey> hoodieKeys, JavaSparkContext jsc, HoodieTable<T> hoodieTable) {
- return null;
- }
-
- @Override
- public JavaRDD<HoodieRecord<T>> tagLocation(JavaRDD<HoodieRecord<T>> recordRDD, JavaSparkContext jsc, HoodieTable<T> hoodieTable) throws HoodieIndexException {
- return null;
- }
-
- @Override
- public JavaRDD<WriteStatus> updateLocation(JavaRDD<WriteStatus> writeStatusRDD, JavaSparkContext jsc, HoodieTable<T> hoodieTable) throws HoodieIndexException {
- return null;
- }
-
- @Override
- public boolean rollbackCommit(String instantTime) {
- return false;
- }
-
- @Override
- public boolean isGlobal() {
- return false;
- }
-
- @Override
- public boolean canIndexLogFiles() {
- return false;
- }
-
- @Override
- public boolean isImplicitWithStorage() {
- return false;
- }
- }
-
- public static class IndexWithConstructor {
-
- public IndexWithConstructor(HoodieWriteConfig config) {
- }
- }
-
- public static class IndexWithoutConstructor {
-
- }
-
}
diff --git a/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java
new file mode 100644
index 0000000..a68a872
--- /dev/null
+++ b/hudi-client/src/test/java/org/apache/hudi/index/TestHoodieIndexConfigs.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.index;
+
+import org.apache.hudi.client.WriteStatus;
+import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.common.model.HoodieRecord;
+import org.apache.hudi.common.model.HoodieRecordPayload;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieHBaseIndexConfig;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.exception.HoodieException;
+import org.apache.hudi.exception.HoodieIndexException;
+import org.apache.hudi.index.HoodieIndex.IndexType;
+import org.apache.hudi.index.bloom.HoodieBloomIndex;
+import org.apache.hudi.index.bloom.HoodieGlobalBloomIndex;
+import org.apache.hudi.index.hbase.HBaseIndex;
+import org.apache.hudi.index.simple.HoodieSimpleIndex;
+import org.apache.hudi.table.HoodieTable;
+
+import org.apache.spark.api.java.JavaPairRDD;
+import org.apache.spark.api.java.JavaRDD;
+import org.apache.spark.api.java.JavaSparkContext;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.nio.file.Path;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+public class TestHoodieIndexConfigs {
+
+ private String basePath;
+
+ @BeforeEach
+ public void setUp(@TempDir Path tempDir) {
+ basePath = tempDir.toString();
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = IndexType.class, names = {"BLOOM", "GLOBAL_BLOOM", "SIMPLE", "GLOBAL_SIMPLE", "HBASE"})
+ public void testCreateIndex(IndexType indexType) throws Exception {
+ HoodieWriteConfig config;
+ HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
+ HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
+ switch (indexType) {
+ case INMEMORY:
+ config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.INMEMORY).build()).build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof InMemoryHashIndex);
+ break;
+ case BLOOM:
+ config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.BLOOM).build()).build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof HoodieBloomIndex);
+ break;
+ case GLOBAL_BLOOM:
+ config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.GLOBAL_BLOOM).build()).build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof HoodieGlobalBloomIndex);
+ break;
+ case SIMPLE:
+ config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexType(IndexType.SIMPLE).build()).build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof HoodieSimpleIndex);
+ break;
+ case HBASE:
+ config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexType(HoodieIndex.IndexType.HBASE)
+ .withHBaseIndexConfig(new HoodieHBaseIndexConfig.Builder().build()).build())
+ .build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof HBaseIndex);
+ break;
+ default:
+ // no -op. just for checkstyle errors
+ }
+ }
+
+ @Test
+ public void testCreateDummyIndex() {
+ HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
+ HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
+ HoodieWriteConfig config = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexClass(DummyHoodieIndex.class.getName()).build()).build();
+ assertTrue(HoodieIndex.createIndex(config) instanceof DummyHoodieIndex);
+ }
+
+ @Test
+ public void testCreateIndexWithException() {
+ HoodieWriteConfig.Builder clientConfigBuilder = HoodieWriteConfig.newBuilder();
+ HoodieIndexConfig.Builder indexConfigBuilder = HoodieIndexConfig.newBuilder();
+ final HoodieWriteConfig config1 = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithConstructor.class.getName()).build()).build();
+ final Throwable thrown1 = assertThrows(HoodieException.class, () -> {
+ HoodieIndex.createIndex(config1);
+ }, "exception is expected");
+ assertTrue(thrown1.getMessage().contains("is not a subclass of HoodieIndex"));
+
+ final HoodieWriteConfig config2 = clientConfigBuilder.withPath(basePath)
+ .withIndexConfig(indexConfigBuilder.withIndexClass(IndexWithoutConstructor.class.getName()).build()).build();
+ final Throwable thrown2 = assertThrows(HoodieException.class, () -> {
+ HoodieIndex.createIndex(config2);
+ }, "exception is expected");
+ assertTrue(thrown2.getMessage().contains("Unable to instantiate class"));
+ }
+
+ public static class DummyHoodieIndex<T extends HoodieRecordPayload<T>> extends HoodieIndex<T> {
+
+ public DummyHoodieIndex(HoodieWriteConfig config) {
+ super(config);
+ }
+
+ @Override
+ public JavaPairRDD<HoodieKey, Option<Pair<String, String>>> fetchRecordLocation(JavaRDD<HoodieKey> hoodieKeys, JavaSparkContext jsc, HoodieTable<T> hoodieTable) {
+ return null;
+ }
+
+ @Override
+ public JavaRDD<HoodieRecord<T>> tagLocation(JavaRDD<HoodieRecord<T>> recordRDD, JavaSparkContext jsc, HoodieTable<T> hoodieTable) throws HoodieIndexException {
+ return null;
+ }
+
+ @Override
+ public JavaRDD<WriteStatus> updateLocation(JavaRDD<WriteStatus> writeStatusRDD, JavaSparkContext jsc, HoodieTable<T> hoodieTable) throws HoodieIndexException {
+ return null;
+ }
+
+ @Override
+ public boolean rollbackCommit(String instantTime) {
+ return false;
+ }
+
+ @Override
+ public boolean isGlobal() {
+ return false;
+ }
+
+ @Override
+ public boolean canIndexLogFiles() {
+ return false;
+ }
+
+ @Override
+ public boolean isImplicitWithStorage() {
+ return false;
+ }
+ }
+
+ public static class IndexWithConstructor {
+
+ public IndexWithConstructor(HoodieWriteConfig config) {
+ }
+ }
+
+ public static class IndexWithoutConstructor {
+
+ }
+}