You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vi...@apache.org on 2025/12/16 19:16:33 UTC
(hudi) branch master updated: refactor: code sweep on hudi-hadoop-common, hudi-common on class organization (#17611)
This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new f973ae6a145a refactor: code sweep on hudi-hadoop-common, hudi-common on class organization (#17611)
f973ae6a145a is described below
commit f973ae6a145a2532613b4db0faa307671e88b9f4
Author: vinoth chandar <vi...@users.noreply.github.com>
AuthorDate: Tue Dec 16 11:16:25 2025 -0800
refactor: code sweep on hudi-hadoop-common, hudi-common on class organization (#17611)
* refactor: code sweep on hudi-hadoop-common, hudi-common on class organization
- Move HoodieParquetConfig to common.config.*
- Reference avro schema convertor class names in code, so usage is IDE traceable
* Remove unused methods in ParquetTableSchemaResolver
* Moving HadoopSeekableDataInputStream to io.hadoop.*
* refactor: Move Hadoop IO classes to io.storage.hadoop package
Relocate 9 classes (HoodieAvroFileReaderFactory, HoodieAvroFileWriterFactory,
HoodieHadoopIOFactory, HoodieAvroParquetReader, HoodieAvroOrcReader,
HoodieAvroParquetWriter, HoodieAvroHFileWriter, HoodieAvroOrcWriter,
HoodieParquetStreamWriter) from io.hadoop to io.storage.hadoop to follow
Java package convention where implementations mirror base package structure.
* moving HoodieAvroParquetReaderIterator to common.util
* Minor changes to add use class names from code and lombok zero-arg constructor
* Clean up ExceptionUtil and SchemaNotFoundException package
* Change value for default HOODIE_IO_FACTORY_CLASS in storage config
---
.../timeline/versioning/v2/LSMTimelineWriter.java | 2 +-
.../hudi/avro/TestHoodieAvroParquetWriter.java | 4 +-
.../hudi/testutils/HoodieWriteableTestTable.java | 6 +--
.../row/HoodieRowDataFileWriterFactory.java | 2 +-
.../HoodieRowDataParquetOutputStreamWriter.java | 2 +-
.../io/storage/row/HoodieRowDataParquetWriter.java | 2 +-
.../hudi/io/storage/HoodieSparkIOFactory.java | 2 +-
.../row/HoodieInternalRowFileWriterFactory.java | 2 +-
.../row/HoodieInternalRowParquetWriter.java | 2 +-
.../io/storage/row/HoodieRowParquetConfig.java | 2 +-
.../hudi/execution/TestDisruptorMessageQueue.java | 2 +-
.../storage/TestHoodieAvroFileWriterFactory.java | 6 +--
.../main/java/org/apache/hudi/ParquetAdapter.java | 6 ++-
.../java/org/apache/hudi/common/bloom/Key.java | 17 ++-------
.../hudi/common/bootstrap/FileStatusUtils.java | 39 --------------------
.../common/bootstrap/index/NoOpBootstrapIndex.java | 2 +-
.../hudi/common/config/HoodieStorageConfig.java | 2 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 11 ++++++
.../common/model/BootstrapBaseFileMapping.java | 11 +++---
.../hudi/common/table/TableSchemaResolver.java | 2 +-
.../org/apache/hudi/exception/ExceptionUtil.java | 43 ----------------------
.../HoodieSchemaNotFoundException.java | 2 +-
.../apache/hudi/util/HoodieFileMetadataMerger.java | 10 ++---
.../hudi/common/testutils/HoodieTestUtils.java | 15 ++++++++
.../hudi/table/format/HoodieFlinkIOFactory.java | 2 +-
.../config}/HoodieParquetConfig.java | 2 +-
.../common/table/ParquetTableSchemaResolver.java | 27 --------------
.../util}/HoodieAvroParquetReaderIterator.java | 3 +-
.../hadoop}/HadoopSeekableDataInputStream.java | 2 +-
.../hudi/io/hadoop/HoodieBaseParquetWriter.java | 2 +-
.../hadoop/HoodieAvroFileReaderFactory.java | 4 +-
.../hadoop/HoodieAvroFileWriterFactory.java | 6 +--
.../hadoop/HoodieAvroHFileWriter.java | 2 +-
.../{ => storage}/hadoop/HoodieAvroOrcReader.java | 3 +-
.../{ => storage}/hadoop/HoodieAvroOrcWriter.java | 2 +-
.../hadoop/HoodieAvroParquetReader.java | 3 +-
.../hadoop/HoodieAvroParquetWriter.java | 5 ++-
.../hadoop/HoodieHadoopIOFactory.java | 4 +-
.../hadoop/HoodieParquetStreamWriter.java | 6 +--
.../hudi/storage/hadoop/HoodieHadoopStorage.java | 2 +-
.../avro/HoodieAvroParquetSchemaConverter.java | 4 +-
.../TestInLineFileSystemWithHFileReader.java | 2 +-
.../io/hadoop/TestHoodieAvroFileReaderFactory.java | 2 +
.../io/hadoop/TestHoodieBaseParquetWriter.java | 2 +-
.../io/hadoop/TestHoodieHFileReaderWriter.java | 1 +
.../hudi/io/hadoop/TestHoodieHadoopIOFactory.java | 1 +
.../hudi/io/hadoop/TestHoodieOrcReaderWriter.java | 1 +
.../TestHoodieNativeAvroHFileReaderCaching.java | 2 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 3 +-
.../org/apache/spark/sql/hudi/SparkHelpers.scala | 6 +--
.../hudi/client/TestHoodieClientMultiWriter.java | 2 +-
.../TestRDDSimpleBucketBulkInsertPartitioner.java | 2 +-
.../org/apache/hudi/functional/TestBootstrap.java | 2 +-
...SparkBinaryCopyClusteringAndValidationMeta.java | 4 +-
.../row/TestHoodieInternalRowParquetWriter.java | 2 +-
.../TestAutoGenerationOfRecordKeys.scala | 6 +--
.../apache/hudi/functional/TestCOWDataSource.scala | 6 +--
.../hudi/functional/TestTimeTravelQuery.scala | 7 ++--
.../sql/hudi/common/HoodieSparkSqlTestBase.scala | 9 ++---
.../deltastreamer/TestHoodieDeltaStreamer.java | 2 +-
60 files changed, 121 insertions(+), 214 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java
index 2ff9c85b8774..7641ab5ad02b 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/timeline/versioning/v2/LSMTimelineWriter.java
@@ -40,7 +40,7 @@ import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieCommitException;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetReader;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetReader;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.HoodieIOFactory;
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
index 523c1afd2919..59797f58bb3a 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroParquetWriter.java
@@ -21,13 +21,13 @@ package org.apache.hudi.avro;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.engine.LocalTaskContextSupplier;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
index 8d8fe7086900..02e939fc7019 100644
--- a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
+++ b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/testutils/HoodieWriteableTestTable.java
@@ -22,6 +22,7 @@ package org.apache.hudi.testutils;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.engine.HoodieEngineContext;
import org.apache.hudi.common.engine.TaskContextSupplier;
@@ -38,10 +39,9 @@ import org.apache.hudi.common.testutils.HoodieMetadataTestTable;
import org.apache.hudi.common.util.CollectionUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.collection.Pair;
-import org.apache.hudi.io.hadoop.HoodieAvroOrcWriter;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.io.storage.HoodieOrcConfig;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.metadata.HoodieTableMetadataWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
index 576b88e3390e..39ec2d08eb18 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataFileWriterFactory.java
@@ -20,6 +20,7 @@ package org.apache.hudi.io.storage.row;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.schema.HoodieSchema;
@@ -28,7 +29,6 @@ import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetOutputStreamWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetOutputStreamWriter.java
index ce6c8c40281c..46b93df8d22c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetOutputStreamWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetOutputStreamWriter.java
@@ -18,9 +18,9 @@
package org.apache.hudi.io.storage.row;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.util.ValidationUtils;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.parquet.io.OutputStreamBackedOutputFile;
import org.apache.flink.table.data.RowData;
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
index 2841d00ac673..1be1fd689681 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataParquetWriter.java
@@ -19,11 +19,11 @@
package org.apache.hudi.io.storage.row;
import org.apache.hudi.client.model.HoodieRowDataCreation;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.flink.table.data.RowData;
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
index 55a0aad05f21..9c596c0f7c2e 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/HoodieSparkIOFactory.java
@@ -20,7 +20,7 @@
package org.apache.hudi.io.storage;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.io.hadoop.HoodieHadoopIOFactory;
+import org.apache.hudi.io.storage.hadoop.HoodieHadoopIOFactory;
import org.apache.hudi.storage.HoodieStorage;
/**
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
index 7ebcd1f39ff8..d51b48f3cd55 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowFileWriterFactory.java
@@ -20,10 +20,10 @@ package org.apache.hudi.io.storage.row;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
index f7ad33d2cbb2..3a0a755affa8 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieInternalRowParquetWriter.java
@@ -18,8 +18,8 @@
package org.apache.hudi.io.storage.row;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.spark.sql.catalyst.InternalRow;
diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetConfig.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetConfig.java
index f3b0f34b929c..1a6782a14800 100644
--- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetConfig.java
+++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowParquetConfig.java
@@ -18,7 +18,7 @@
package org.apache.hudi.io.storage.row;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.hadoop.conf.Configuration;
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java
index efc403a8d5a5..d796ffde358a 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/execution/TestDisruptorMessageQueue.java
@@ -57,7 +57,7 @@ import java.util.stream.IntStream;
import scala.Tuple2;
-import static org.apache.hudi.exception.ExceptionUtil.getRootCause;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.getRootCause;
import static org.apache.hudi.execution.HoodieLazyInsertIterable.getTransformerInternal;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
index 92b1dc9a0f39..0d9734da1285 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/io/storage/TestHoodieAvroFileWriterFactory.java
@@ -23,9 +23,9 @@ import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.index.HoodieIndex.IndexType;
-import org.apache.hudi.io.hadoop.HoodieAvroHFileWriter;
-import org.apache.hudi.io.hadoop.HoodieAvroOrcWriter;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroHFileWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-common/src/main/java/org/apache/hudi/ParquetAdapter.java b/hudi-common/src/main/java/org/apache/hudi/ParquetAdapter.java
index a495fd83afab..62caf072597f 100644
--- a/hudi-common/src/main/java/org/apache/hudi/ParquetAdapter.java
+++ b/hudi-common/src/main/java/org/apache/hudi/ParquetAdapter.java
@@ -22,6 +22,8 @@ package org.apache.hudi;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.stats.ValueType;
+import org.apache.parquet.schema.LogicalTypeParquetAdapter;
+import org.apache.parquet.schema.OriginalTypeParquetAdapter;
import org.apache.parquet.schema.PrimitiveType;
/**
@@ -32,9 +34,9 @@ public interface ParquetAdapter {
static ParquetAdapter getAdapter() {
try {
- return ReflectionUtils.loadClass("org.apache.parquet.schema.LogicalTypeParquetAdapter");
+ return ReflectionUtils.loadClass(LogicalTypeParquetAdapter.class.getName());
} catch (Throwable t) {
- return ReflectionUtils.loadClass("org.apache.parquet.schema.OriginalTypeParquetAdapter");
+ return ReflectionUtils.loadClass(OriginalTypeParquetAdapter.class.getName());
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
index f14d301ae3b3..013c0f08ea46 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/bloom/Key.java
@@ -19,6 +19,8 @@
package org.apache.hudi.common.bloom;
+import lombok.Getter;
+
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -30,6 +32,7 @@ import java.io.IOException;
*
* @see InternalBloomFilter The general behavior of a bloom filter and how the key is used.
*/
+@Getter
public class Key implements Comparable<Key> {
/**
* Byte value of key
@@ -85,20 +88,6 @@ public class Key implements Comparable<Key> {
this.weight = weight;
}
- /**
- * @return byte[] The value of <i>this</i> key.
- */
- public byte[] getBytes() {
- return this.bytes;
- }
-
- /**
- * @return Returns the weight associated to <i>this</i> key.
- */
- public double getWeight() {
- return weight;
- }
-
/**
* Increments the weight of <i>this</i> key with a specified value.
*
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/FileStatusUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/FileStatusUtils.java
deleted file mode 100644
index 94df94fe9ebf..000000000000
--- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/FileStatusUtils.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.common.bootstrap;
-
-import org.apache.hudi.avro.model.HoodieFileStatus;
-import org.apache.hudi.storage.StoragePath;
-import org.apache.hudi.storage.StoragePathInfo;
-
-/**
- * Helper functions around FileStatus and HoodieFileStatus.
- */
-public class FileStatusUtils {
- public static StoragePathInfo toStoragePathInfo(HoodieFileStatus fileStatus) {
- if (null == fileStatus) {
- return null;
- }
-
- return new StoragePathInfo(
- new StoragePath(fileStatus.getPath().getUri()), fileStatus.getLength(),
- fileStatus.getIsDir() != null && fileStatus.getIsDir(),
- fileStatus.getBlockReplication().shortValue(), fileStatus.getBlockSize(), fileStatus.getModificationTime());
- }
-}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/NoOpBootstrapIndex.java b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/NoOpBootstrapIndex.java
index 95627a3b71e0..e51b1aa694f9 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/NoOpBootstrapIndex.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/bootstrap/index/NoOpBootstrapIndex.java
@@ -22,7 +22,7 @@ package org.apache.hudi.common.bootstrap.index;
import org.apache.hudi.common.table.HoodieTableMetaClient;
/**
- * No Op Bootstrap Index , which is a empty implement and not do anything.
+ * No Op Bootstrap Index is an empty implementation.
*/
public class NoOpBootstrapIndex extends BootstrapIndex {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
index 82c6b7a64f80..5d90367f913e 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieStorageConfig.java
@@ -264,7 +264,7 @@ public class HoodieStorageConfig extends HoodieConfig {
public static final ConfigProperty<String> HOODIE_IO_FACTORY_CLASS = ConfigProperty
.key("hoodie.io.factory.class")
- .defaultValue("org.apache.hudi.io.hadoop.HoodieHadoopIOFactory")
+ .defaultValue("org.apache.hudi.io.storage.hadoop.HoodieHadoopIOFactory")
.markAdvanced()
.sinceVersion("0.15.0")
.withDocumentation("The fully-qualified class name of the factory class to return readers and writers of files used "
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
index 88d89a94188d..b9d9425db2f2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/fs/FSUtils.java
@@ -739,6 +739,17 @@ public class FSUtils {
return s3aUrl.replaceFirst("(?i)^s3a://", "s3://");
}
+ public static StoragePathInfo toStoragePathInfo(HoodieFileStatus fileStatus) {
+ if (null == fileStatus) {
+ return null;
+ }
+
+ return new StoragePathInfo(
+ new StoragePath(fileStatus.getPath().getUri()), fileStatus.getLength(),
+ fileStatus.getIsDir() != null && fileStatus.getIsDir(),
+ fileStatus.getBlockReplication().shortValue(), fileStatus.getBlockSize(), fileStatus.getModificationTime());
+ }
+
/**
* Serializable function interface.
*
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java
index e01c0356a01e..a8b1a561be5c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/model/BootstrapBaseFileMapping.java
@@ -19,7 +19,9 @@
package org.apache.hudi.common.model;
import org.apache.hudi.avro.model.HoodieFileStatus;
-import org.apache.hudi.common.bootstrap.FileStatusUtils;
+import org.apache.hudi.common.fs.FSUtils;
+
+import lombok.Getter;
import java.io.Serializable;
@@ -28,6 +30,7 @@ import java.io.Serializable;
*/
public class BootstrapBaseFileMapping implements Serializable {
+ @Getter
private final HoodieFileGroupId fileGroupId;
private final HoodieFileStatus bootstrapFileStatus;
@@ -37,12 +40,8 @@ public class BootstrapBaseFileMapping implements Serializable {
this.bootstrapFileStatus = bootstrapFileStatus;
}
- public HoodieFileGroupId getFileGroupId() {
- return fileGroupId;
- }
-
public BaseFile getBootstrapBaseFile() {
- return new BaseFile(FileStatusUtils.toStoragePathInfo(bootstrapFileStatus));
+ return new BaseFile(FSUtils.toStoragePathInfo(bootstrapFileStatus));
}
@Override
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
index 2309ef5ab743..f99e3e2397ec 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/table/TableSchemaResolver.java
@@ -19,7 +19,6 @@
package org.apache.hudi.common.table;
import org.apache.hudi.avro.HoodieAvroUtils;
-import org.apache.hudi.common.HoodieSchemaNotFoundException;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
import org.apache.hudi.common.model.HoodieLogFile;
@@ -41,6 +40,7 @@ import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.exception.HoodieSchemaNotFoundException;
import org.apache.hudi.exception.InvalidTableException;
import org.apache.hudi.internal.schema.HoodieSchemaException;
import org.apache.hudi.internal.schema.InternalSchema;
diff --git a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java b/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java
deleted file mode 100644
index 021c769f8bf2..000000000000
--- a/hudi-common/src/main/java/org/apache/hudi/exception/ExceptionUtil.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hudi.exception;
-
-import javax.annotation.Nonnull;
-
-/**
- * Class collecting common utilities helping in handling {@link Exception}s
- */
-public final class ExceptionUtil {
- private ExceptionUtil() {
- }
-
- /**
- * Fetches inner-most cause of the provided {@link Throwable}
- */
- @Nonnull
- public static Throwable getRootCause(@Nonnull Throwable t) {
- Throwable cause = t;
- while (cause.getCause() != null) {
- cause = cause.getCause();
- }
-
- return cause;
- }
-
-}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/HoodieSchemaNotFoundException.java b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSchemaNotFoundException.java
similarity index 96%
rename from hudi-common/src/main/java/org/apache/hudi/common/HoodieSchemaNotFoundException.java
rename to hudi-common/src/main/java/org/apache/hudi/exception/HoodieSchemaNotFoundException.java
index 12d1498b9740..959852c5e0b7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/HoodieSchemaNotFoundException.java
+++ b/hudi-common/src/main/java/org/apache/hudi/exception/HoodieSchemaNotFoundException.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.common;
+package org.apache.hudi.exception;
import org.apache.hudi.internal.schema.HoodieSchemaException;
diff --git a/hudi-common/src/main/java/org/apache/hudi/util/HoodieFileMetadataMerger.java b/hudi-common/src/main/java/org/apache/hudi/util/HoodieFileMetadataMerger.java
index ee7ae3f815df..bb4ce5eb0bf7 100644
--- a/hudi-common/src/main/java/org/apache/hudi/util/HoodieFileMetadataMerger.java
+++ b/hudi-common/src/main/java/org/apache/hudi/util/HoodieFileMetadataMerger.java
@@ -23,25 +23,21 @@ import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.util.StringUtils;
+import lombok.NoArgsConstructor;
+
import java.util.HashMap;
import java.util.Map;
import static org.apache.hudi.avro.HoodieBloomFilterWriteSupport.HOODIE_AVRO_BLOOM_FILTER_METADATA_KEY;
+@NoArgsConstructor
public class HoodieFileMetadataMerger {
private final HashMap<String, String> mergedMetadata = new HashMap<>();
-
private BloomFilter bloomFilter;
-
private String minRecordKey;
-
private String maxRecordKey;
- public HoodieFileMetadataMerger() {
-
- }
-
public Map<String, String> mergeMetaData(Map<String, String> metaMap) {
mergedMetadata.putAll(metaMap);
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
index cc5bfd3a2d96..99ea61ffb593 100644
--- a/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/testutils/HoodieTestUtils.java
@@ -68,6 +68,8 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.junit.jupiter.api.Assumptions;
+import javax.annotation.Nonnull;
+
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -533,4 +535,17 @@ public class HoodieTestUtils {
nonExistentConfigs.forEach(key -> assertFalse(
tableConfig.contains(key), key + " should not be present in the table config"));
}
+
+ /**
+ * Fetches inner-most cause of the provided {@link Throwable}
+ */
+ @Nonnull
+ public static Throwable getRootCause(@Nonnull Throwable t) {
+ Throwable cause = t;
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ }
+
+ return cause;
+ }
}
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieFlinkIOFactory.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieFlinkIOFactory.java
index 1ed7ee927984..da90d615df74 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieFlinkIOFactory.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/table/format/HoodieFlinkIOFactory.java
@@ -19,7 +19,7 @@
package org.apache.hudi.table.format;
import org.apache.hudi.common.model.HoodieRecord;
-import org.apache.hudi.io.hadoop.HoodieHadoopIOFactory;
+import org.apache.hudi.io.storage.hadoop.HoodieHadoopIOFactory;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.row.HoodieRowDataFileWriterFactory;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/HoodieParquetConfig.java
similarity index 98%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/HoodieParquetConfig.java
index e17a017d6797..608b56234748 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieParquetConfig.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/config/HoodieParquetConfig.java
@@ -16,7 +16,7 @@
* limitations under the License.
*/
-package org.apache.hudi.io.storage;
+package org.apache.hudi.common.config;
import org.apache.hudi.storage.StorageConfiguration;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/ParquetTableSchemaResolver.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/ParquetTableSchemaResolver.java
index 066d381f23f6..08ec8f24654c 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/ParquetTableSchemaResolver.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/table/ParquetTableSchemaResolver.java
@@ -34,31 +34,4 @@ public class ParquetTableSchemaResolver extends TableSchemaResolver {
public static MessageType convertAvroSchemaToParquet(Schema schema, Configuration hadoopConf) {
return getAvroSchemaConverter(hadoopConf).convert(schema);
}
-
- private Schema convertParquetSchemaToAvro(MessageType parquetSchema) {
- return getAvroSchemaConverter(metaClient.getStorageConf().unwrapAs(Configuration.class)).convert(parquetSchema);
- }
-
- private MessageType convertAvroSchemaToParquet(Schema schema) {
- return getAvroSchemaConverter(metaClient.getStorageConf().unwrapAs(Configuration.class)).convert(schema);
- }
-
- /**
- * Gets full schema (user + metadata) for a hoodie table in Parquet format.
- *
- * @return Parquet schema for the table
- */
- public MessageType getTableParquetSchema() throws Exception {
- return convertAvroSchemaToParquet(getTableAvroSchema(true));
- }
-
- /**
- * Gets users data schema for a hoodie table in Parquet format.
- *
- * @return Parquet schema for the table
- */
- public MessageType getTableParquetSchema(boolean includeMetadataField) throws Exception {
- return convertAvroSchemaToParquet(getTableAvroSchema(includeMetadataField));
- }
-
}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HoodieAvroParquetReaderIterator.java
similarity index 94%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HoodieAvroParquetReaderIterator.java
index c0eaf16d08b5..8a72d48609cd 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReaderIterator.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HoodieAvroParquetReaderIterator.java
@@ -17,10 +17,9 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.common.util;
import org.apache.hudi.avro.HoodieAvroUtils;
-import org.apache.hudi.common.util.ParquetReaderIterator;
import org.apache.avro.Schema;
import org.apache.avro.generic.IndexedRecord;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopSeekableDataInputStream.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HadoopSeekableDataInputStream.java
similarity index 97%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopSeekableDataInputStream.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HadoopSeekableDataInputStream.java
index ae10ca0ac3f6..15ceac92927c 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/hadoop/fs/HadoopSeekableDataInputStream.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HadoopSeekableDataInputStream.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.hadoop.fs;
+package org.apache.hudi.io.hadoop;
import org.apache.hudi.io.SeekableDataInputStream;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
index 24ad77a6c135..ffacac30ca95 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieBaseParquetWriter.java
@@ -19,10 +19,10 @@
package org.apache.hudi.io.hadoop;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.util.VisibleForTesting;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.hadoop.conf.Configuration;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileReaderFactory.java
similarity index 98%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileReaderFactory.java
index c7dcbf5463c0..ce670c51114e 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileReaderFactory.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.common.config.HoodieConfig;
import org.apache.hudi.common.schema.HoodieSchema;
@@ -86,4 +86,4 @@ public class HoodieAvroFileReaderFactory extends HoodieFileReaderFactory {
public HoodieFileReader newBootstrapFileReader(HoodieFileReader skeletonFileReader, HoodieFileReader dataFileReader, Option<String[]> partitionFields, Object[] partitionValues) {
return new HoodieAvroBootstrapFileReader(skeletonFileReader, dataFileReader, partitionFields, partitionValues);
}
-}
\ No newline at end of file
+}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java
similarity index 98%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java
index 741e7c18ce73..2d16d6343d27 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroFileWriterFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroFileWriterFactory.java
@@ -17,11 +17,12 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.schema.HoodieSchema;
@@ -34,7 +35,6 @@ import org.apache.hudi.io.storage.HoodieFileWriter;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.HoodieHFileConfig;
import org.apache.hudi.io.storage.HoodieOrcConfig;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
@@ -126,4 +126,4 @@ public class HoodieAvroFileWriterFactory extends HoodieFileWriterFactory {
new Class<?>[] {MessageType.class, Schema.class, Option.class, Properties.class},
getAvroSchemaConverter(storage.getConf().unwrapAs(Configuration.class)).convert(schema), schema, filter, config.getProps());
}
-}
\ No newline at end of file
+}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
similarity index 99%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
index ddc58d5a118e..ca46018e0d17 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroHFileWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroHFileWriter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.bloom.BloomFilter;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java
similarity index 98%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java
index 2be916884f50..d1b5e0e98b26 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.bloom.BloomFilter;
@@ -30,6 +30,7 @@ import org.apache.hudi.common.util.collection.ClosableIterator;
import org.apache.hudi.common.util.collection.CloseableMappingIterator;
import org.apache.hudi.common.util.collection.Pair;
import org.apache.hudi.exception.HoodieIOException;
+import org.apache.hudi.io.hadoop.OrcReaderIterator;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java
similarity index 99%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcWriter.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java
index e3ffc69c1a84..61710c67636a 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroOrcWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroOrcWriter.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetReader.java
similarity index 99%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetReader.java
index 902a15bd03ef..9f7bc80cd894 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetReader.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetReader.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.bloom.BloomFilter;
@@ -27,6 +27,7 @@ import org.apache.hudi.common.model.HoodieRecord;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.schema.HoodieSchemaUtils;
import org.apache.hudi.common.util.FileFormatUtils;
+import org.apache.hudi.common.util.HoodieAvroParquetReaderIterator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetReaderIterator;
import org.apache.hudi.common.util.collection.ClosableIterator;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java
similarity index 95%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetWriter.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java
index f8f9a8ccea0f..93f15f54893e 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieAvroParquetWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieAvroParquetWriter.java
@@ -17,13 +17,14 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.engine.TaskContextSupplier;
import org.apache.hudi.common.model.HoodieKey;
+import org.apache.hudi.io.hadoop.HoodieBaseParquetWriter;
import org.apache.hudi.io.storage.HoodieAvroFileWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.avro.generic.IndexedRecord;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieHadoopIOFactory.java
similarity index 99%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieHadoopIOFactory.java
index 472d804d3267..3d8e797682e9 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHadoopIOFactory.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieHadoopIOFactory.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.common.fs.ConsistencyGuard;
import org.apache.hudi.common.model.HoodieFileFormat;
@@ -132,4 +132,4 @@ public class HoodieHadoopIOFactory extends HoodieIOFactory {
return new HoodieHadoopStorage(path, storage.getConf(), enableRetry, maxRetryIntervalMs,
maxRetryNumbers, maxRetryIntervalMs, retryExceptions, consistencyGuard);
}
-}
\ No newline at end of file
+}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieParquetStreamWriter.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieParquetStreamWriter.java
similarity index 97%
rename from hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieParquetStreamWriter.java
rename to hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieParquetStreamWriter.java
index 5fdd6505733f..dacbb14ac2ba 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieParquetStreamWriter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/hadoop/HoodieParquetStreamWriter.java
@@ -17,12 +17,12 @@
* under the License.
*/
-package org.apache.hudi.io.hadoop;
+package org.apache.hudi.io.storage.hadoop;
import org.apache.hudi.avro.HoodieAvroWriteSupport;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.model.HoodieKey;
import org.apache.hudi.io.storage.HoodieAvroFileWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.parquet.io.OutputStreamBackedOutputFile;
import org.apache.avro.generic.IndexedRecord;
@@ -106,4 +106,4 @@ public class HoodieParquetStreamWriter implements HoodieAvroFileWriter, AutoClos
return writeSupport;
}
}
-}
\ No newline at end of file
+}
diff --git a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
index 6d5549c7ef98..87a9ad1019f6 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/hudi/storage/hadoop/HoodieHadoopStorage.java
@@ -22,10 +22,10 @@ package org.apache.hudi.storage.hadoop;
import org.apache.hudi.common.fs.ConsistencyGuard;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
-import org.apache.hudi.hadoop.fs.HadoopSeekableDataInputStream;
import org.apache.hudi.hadoop.fs.HoodieRetryWrapperFileSystem;
import org.apache.hudi.hadoop.fs.HoodieWrapperFileSystem;
import org.apache.hudi.io.SeekableDataInputStream;
+import org.apache.hudi.io.hadoop.HadoopSeekableDataInputStream;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetSchemaConverter.java b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetSchemaConverter.java
index 5473afd36d83..853b186f19df 100644
--- a/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetSchemaConverter.java
+++ b/hudi-hadoop-common/src/main/java/org/apache/parquet/avro/HoodieAvroParquetSchemaConverter.java
@@ -42,11 +42,11 @@ public abstract class HoodieAvroParquetSchemaConverter {
private static final Logger LOG = LoggerFactory.getLogger(HoodieAvroParquetSchemaConverter.class);
public static HoodieAvroParquetSchemaConverter getAvroSchemaConverter(Configuration configuration) {
try {
- return (HoodieAvroParquetSchemaConverter) ReflectionUtils.loadClass("org.apache.parquet.avro.AvroSchemaConverterWithTimestampNTZ",
+ return (HoodieAvroParquetSchemaConverter) ReflectionUtils.loadClass(AvroSchemaConverterWithTimestampNTZ.class.getName(),
new Class<?>[] {Configuration.class}, configuration);
} catch (Throwable t) {
LOG.debug("Failed to load AvroSchemaConverterWithTimestampNTZ, using NativeAvroSchemaConverter instead", t);
- return (HoodieAvroParquetSchemaConverter) ReflectionUtils.loadClass("org.apache.parquet.avro.NativeAvroSchemaConverter",
+ return (HoodieAvroParquetSchemaConverter) ReflectionUtils.loadClass(NativeAvroSchemaConverter.class.getName(),
new Class<?>[] {Configuration.class}, configuration);
}
}
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java
index f5a45f11ab84..4979d54a311a 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHFileReader.java
@@ -21,10 +21,10 @@ package org.apache.hudi.common.fs.inline;
import org.apache.hudi.common.testutils.FileSystemTestUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.hadoop.fs.HadoopSeekableDataInputStream;
import org.apache.hudi.hadoop.fs.inline.InLineFileSystem;
import org.apache.hudi.hadoop.fs.inline.InMemoryFileSystem;
import org.apache.hudi.io.SeekableDataInputStream;
+import org.apache.hudi.io.hadoop.HadoopSeekableDataInputStream;
import org.apache.hudi.io.hfile.HFileContext;
import org.apache.hudi.io.hfile.HFileReader;
import org.apache.hudi.io.hfile.HFileReaderImpl;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
index 1bb2324505a9..337ad4d2b8f9 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieAvroFileReaderFactory.java
@@ -24,6 +24,8 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieFileReader;
import org.apache.hudi.io.storage.HoodieFileReaderFactory;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcReader;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetReader;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieBaseParquetWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieBaseParquetWriter.java
index 82a80b1ce262..6bcd2b473fb4 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieBaseParquetWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieBaseParquetWriter.java
@@ -23,10 +23,10 @@ import org.apache.hudi.avro.HoodieAvroWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.Option;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
index 42111d28c159..4decc40e2770 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriter.java
@@ -43,6 +43,7 @@ import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieAvroHFileReaderImplBase;
import org.apache.hudi.io.storage.HoodieFileWriterFactory;
import org.apache.hudi.io.storage.HoodieNativeAvroHFileReader;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroHFileWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
index 23599d9271de..4447b8ff3ee5 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHadoopIOFactory.java
@@ -25,6 +25,7 @@ import org.apache.hudi.common.util.OrcUtils;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.io.storage.HoodieIOFactory;
+import org.apache.hudi.io.storage.hadoop.HoodieHadoopIOFactory;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HoodieHadoopStorage;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
index 0b735ce9f165..522e3d7e65a6 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieOrcReaderWriter.java
@@ -30,6 +30,7 @@ import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.io.storage.HoodieAvroFileReader;
import org.apache.hudi.io.storage.HoodieIOFactory;
import org.apache.hudi.io.storage.HoodieOrcConfig;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroOrcWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StorageConfiguration;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
index be35301333c1..227fa629b17f 100644
--- a/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
+++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/io/storage/TestHoodieNativeAvroHFileReaderCaching.java
@@ -31,7 +31,7 @@ import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.testutils.HoodieTestUtils;
import org.apache.hudi.common.util.collection.ClosableIterator;
-import org.apache.hudi.io.hadoop.HoodieAvroHFileWriter;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroHFileWriter;
import org.apache.hudi.io.hadoop.TestHoodieOrcReaderWriter;
import org.apache.hudi.storage.HoodieStorage;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
index 363aa116f7fa..ddc176dff26f 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/DefaultSource.scala
@@ -20,7 +20,6 @@ package org.apache.hudi
import org.apache.hudi.DataSourceReadOptions._
import org.apache.hudi.DataSourceWriteOptions.{BOOTSTRAP_OPERATION_OPT_VAL, OPERATION, STREAMING_CHECKPOINT_IDENTIFIER}
import org.apache.hudi.cdc.HoodieCDCFileIndex
-import org.apache.hudi.common.HoodieSchemaNotFoundException
import org.apache.hudi.common.config.HoodieStorageConfig
import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, MERGE_ON_READ}
import org.apache.hudi.common.model.WriteConcurrencyMode
@@ -28,7 +27,7 @@ import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, H
import org.apache.hudi.common.util.{ConfigUtils, TablePathUtils}
import org.apache.hudi.common.util.ValidationUtils.checkState
import org.apache.hudi.config.HoodieWriteConfig.{WRITE_CONCURRENCY_MODE, WRITE_TABLE_VERSION}
-import org.apache.hudi.exception.HoodieException
+import org.apache.hudi.exception.{HoodieException, HoodieSchemaNotFoundException}
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.io.storage.HoodieSparkIOFactory
import org.apache.hudi.storage.{HoodieStorageUtils, StoragePath}
diff --git a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
index 1344a738c956..a944a7d43661 100644
--- a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
+++ b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/SparkHelpers.scala
@@ -20,12 +20,12 @@ package org.apache.spark.sql.hudi
import org.apache.hudi.avro.HoodieAvroWriteSupport
import org.apache.hudi.client.SparkTaskContextSupplier
import org.apache.hudi.common.bloom.{BloomFilter, BloomFilterFactory}
-import org.apache.hudi.common.config.HoodieStorageConfig
+import org.apache.hudi.common.config.{HoodieParquetConfig, HoodieStorageConfig}
import org.apache.hudi.common.config.HoodieStorageConfig.{BLOOM_FILTER_DYNAMIC_MAX_ENTRIES, BLOOM_FILTER_FPP_VALUE, BLOOM_FILTER_NUM_ENTRIES_VALUE, BLOOM_FILTER_TYPE}
import org.apache.hudi.common.model.{HoodieFileFormat, HoodieRecord}
import org.apache.hudi.common.util.Option
-import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter
-import org.apache.hudi.io.storage.{HoodieIOFactory, HoodieParquetConfig}
+import org.apache.hudi.io.storage.HoodieIOFactory
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetWriter
import org.apache.hudi.storage.{HoodieStorage, StorageConfiguration, StoragePath}
import org.apache.avro.Schema
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
index 3392074c204b..e502e8aebde0 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/client/TestHoodieClientMultiWriter.java
@@ -24,7 +24,6 @@ import org.apache.hudi.client.transaction.SimpleConcurrentFileWritesConflictReso
import org.apache.hudi.client.transaction.lock.FileSystemBasedLockProvider;
import org.apache.hudi.client.transaction.lock.InProcessLockProvider;
import org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider;
-import org.apache.hudi.common.HoodieSchemaNotFoundException;
import org.apache.hudi.common.config.HoodieMetadataConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.config.LockConfiguration;
@@ -60,6 +59,7 @@ import org.apache.hudi.config.HoodieCompactionConfig;
import org.apache.hudi.config.HoodieLockConfig;
import org.apache.hudi.config.HoodieWriteConfig;
import org.apache.hudi.exception.HoodieSchemaEvolutionConflictException;
+import org.apache.hudi.exception.HoodieSchemaNotFoundException;
import org.apache.hudi.exception.HoodieWriteConflictException;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/execution/bulkinsert/TestRDDSimpleBucketBulkInsertPartitioner.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/execution/bulkinsert/TestRDDSimpleBucketBulkInsertPartitioner.java
index 8b489d1eb239..9f047fc006a7 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/execution/bulkinsert/TestRDDSimpleBucketBulkInsertPartitioner.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/execution/bulkinsert/TestRDDSimpleBucketBulkInsertPartitioner.java
@@ -49,7 +49,7 @@ import java.util.List;
import java.util.Map;
import static org.apache.hudi.common.testutils.HoodieTestDataGenerator.TRIP_EXAMPLE_SCHEMA;
-import static org.apache.hudi.exception.ExceptionUtil.getRootCause;
+import static org.apache.hudi.common.testutils.HoodieTestUtils.getRootCause;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertLinesMatch;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
index c1d79a9a2ea5..c5525d7fdf3b 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestBootstrap.java
@@ -53,7 +53,7 @@ import org.apache.hudi.hadoop.HoodieParquetInputFormat;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hadoop.realtime.HoodieParquetRealtimeInputFormat;
import org.apache.hudi.index.HoodieIndex.IndexType;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetReader;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetReader;
import org.apache.hudi.keygen.NonpartitionedKeyGenerator;
import org.apache.hudi.keygen.SimpleKeyGenerator;
import org.apache.hudi.storage.StoragePath;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
index fab6dce9c889..4d40dd0b698e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/functional/TestSparkBinaryCopyClusteringAndValidationMeta.java
@@ -28,6 +28,7 @@ import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
import org.apache.hudi.common.config.HoodieConfig;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.engine.LocalTaskContextSupplier;
import org.apache.hudi.common.model.ClusteringGroupInfo;
import org.apache.hudi.common.model.ClusteringOperation;
@@ -42,8 +43,7 @@ import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.config.HoodieClusteringConfig;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetWriter;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetWriter;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.table.HoodieSparkTable;
import org.apache.hudi.table.HoodieTable;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
index 06829365cad2..b2c9d24b1b96 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
+++ b/hudi-spark-datasource/hudi-spark/src/test/java/org/apache/hudi/io/storage/row/TestHoodieInternalRowParquetWriter.java
@@ -22,12 +22,12 @@ import org.apache.hudi.avro.HoodieBloomFilterWriteSupport;
import org.apache.hudi.common.bloom.BloomFilter;
import org.apache.hudi.common.bloom.BloomFilterFactory;
import org.apache.hudi.common.bloom.BloomFilterTypeCode;
+import org.apache.hudi.common.config.HoodieParquetConfig;
import org.apache.hudi.common.config.HoodieStorageConfig;
import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ParquetUtils;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.io.storage.HoodieParquetConfig;
import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.hudi.testutils.HoodieSparkClientTestHarness;
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
index 4f88470953f7..b434801ec056 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestAutoGenerationOfRecordKeys.scala
@@ -24,9 +24,9 @@ import org.apache.hudi.HoodieConversionUtils.toJavaOption
import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
import org.apache.hudi.common.model.HoodieRecord.HoodieRecordType
import org.apache.hudi.common.testutils.HoodieTestDataGenerator.recordsToStrings
+import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.util.Option
import org.apache.hudi.exception.{HoodieException, HoodieKeyGeneratorException}
-import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.functional.CommonOptionUtils._
import org.apache.hudi.keygen.{ComplexKeyGenerator, NonpartitionedKeyGenerator, SimpleKeyGenerator, TimestampBasedKeyGenerator}
import org.apache.hudi.keygen.constant.KeyGeneratorOptions
@@ -200,7 +200,7 @@ class TestAutoGenerationOfRecordKeys extends HoodieSparkClientTestBase with Scal
.save(basePath)
}
- assertTrue(getRootCause(e).getMessage.contains(configKey + " is not supported with auto generation of record keys"))
+ assertTrue(HoodieTestUtils.getRootCause(e).getMessage.contains(configKey + " is not supported with auto generation of record keys"))
}
@Test
@@ -245,7 +245,7 @@ class TestAutoGenerationOfRecordKeys extends HoodieSparkClientTestBase with Scal
}
val expectedMsg = s"RecordKey:\t_row_key\tnull"
- assertTrue(getRootCause(e).getMessage.contains(expectedMsg))
+ assertTrue(HoodieTestUtils.getRootCause(e).getMessage.contains(expectedMsg))
}
@Test
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
index 1079abaa8a69..a49591f429b9 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestCOWDataSource.scala
@@ -38,7 +38,6 @@ import org.apache.hudi.common.util.{ClusteringUtils, Option}
import org.apache.hudi.config.HoodieWriteConfig
import org.apache.hudi.config.metrics.HoodieMetricsConfig
import org.apache.hudi.exception.{HoodieException, SchemaBackwardsCompatibilityException}
-import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.hive.HiveSyncConfigHolder
import org.apache.hudi.keygen.{ComplexKeyGenerator, CustomKeyGenerator, GlobalDeleteKeyGenerator, NonpartitionedKeyGenerator, SimpleKeyGenerator, TimestampBasedKeyGenerator}
import org.apache.hudi.keygen.constant.{KeyGeneratorOptions, KeyGeneratorType}
@@ -488,7 +487,7 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
val t = assertThrows(classOf[Throwable]) {
writeToHudi(optsForBatch2, inputDF)
}
- assertTrue(getRootCause(t).getMessage.contains("Config conflict"))
+ assertTrue(HoodieTestUtils.getRootCause(t).getMessage.contains("Config conflict"))
}
@ParameterizedTest
@@ -1313,7 +1312,8 @@ class TestCOWDataSource extends HoodieSparkClientTestBase with ScalaAssertionSup
.save(basePath)
}
- assertEquals("Single partition-path field is expected; provided (driver,rider)", getRootCause(t).getMessage)
+ assertEquals("Single partition-path field is expected; provided (driver,rider)",
+ HoodieTestUtils.getRootCause(t).getMessage)
}
@ParameterizedTest
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
index 93c80f947920..e4eb32814875 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestTimeTravelQuery.scala
@@ -23,9 +23,8 @@ import org.apache.hudi.common.model.{HoodieCleaningPolicy, HoodieTableType}
import org.apache.hudi.common.model.HoodieTableType.{COPY_ON_WRITE, MERGE_ON_READ}
import org.apache.hudi.common.table.{HoodieTableConfig, TableSchemaResolver}
import org.apache.hudi.common.table.timeline.TimelineUtils
-import org.apache.hudi.common.testutils.HoodieTestTable
+import org.apache.hudi.common.testutils.{HoodieTestTable, HoodieTestUtils}
import org.apache.hudi.config.{HoodieArchivalConfig, HoodieCleanConfig, HoodieCompactionConfig, HoodieWriteConfig}
-import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.exception.HoodieTimeTravelException
import org.apache.hudi.testutils.HoodieSparkClientTestBase
@@ -367,7 +366,7 @@ class TestTimeTravelQuery extends HoodieSparkClientTestBase with ScalaAssertionS
.select("id", "name", "value", "version")
.take(1)
}
- assertTrue(getRootCause(e1).getMessage.contains("Cleaner cleaned up the timestamp of interest. Please ensure sufficient commits are retained with cleaner for Timestamp as of query to work"))
+ assertTrue(HoodieTestUtils.getRootCause(e1).getMessage.contains("Cleaner cleaned up the timestamp of interest. Please ensure sufficient commits are retained with cleaner for Timestamp as of query to work"))
}
// add more writes so that first commit goes into archived timeline.
@@ -392,6 +391,6 @@ class TestTimeTravelQuery extends HoodieSparkClientTestBase with ScalaAssertionS
.select("id", "name", "value", "version")
.take(1)
}
- assertTrue(getRootCause(e2).getMessage.contains(expectedErrorMsg))
+ assertTrue(HoodieTestUtils.getRootCause(e2).getMessage.contains(expectedErrorMsg))
}
}
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
index 5a93fcef6e2c..c407ae704869 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/common/HoodieSparkSqlTestBase.scala
@@ -31,7 +31,6 @@ import org.apache.hudi.common.table.view.{FileSystemViewManager, FileSystemViewS
import org.apache.hudi.common.testutils.HoodieTestUtils
import org.apache.hudi.common.util.OrderingValues
import org.apache.hudi.config.HoodieWriteConfig
-import org.apache.hudi.exception.ExceptionUtil.getRootCause
import org.apache.hudi.hadoop.fs.HadoopFSUtils
import org.apache.hudi.index.inmemory.HoodieInMemoryHashIndex
import org.apache.hudi.storage.{HoodieStorage, StoragePath}
@@ -219,7 +218,7 @@ class HoodieSparkSqlTestBase extends FunSuite with BeforeAndAfterAll {
try {
runnable.run()
} catch {
- case e: Throwable if checkMessageContains(e, errorMsg) || checkMessageContains(getRootCause(e), errorMsg) =>
+ case e: Throwable if checkMessageContains(e, errorMsg) || checkMessageContains(HoodieTestUtils.getRootCause(e), errorMsg) =>
hasException = true
case f: Throwable =>
@@ -233,7 +232,7 @@ class HoodieSparkSqlTestBase extends FunSuite with BeforeAndAfterAll {
try {
spark.sql(sql)
} catch {
- case e: Throwable if checkMessageContains(e, errorMsg) || checkMessageContains(getRootCause(e), errorMsg) =>
+ case e: Throwable if checkMessageContains(e, errorMsg) || checkMessageContains(HoodieTestUtils.getRootCause(e), errorMsg) =>
hasException = true
case f: Throwable =>
@@ -247,11 +246,11 @@ class HoodieSparkSqlTestBase extends FunSuite with BeforeAndAfterAll {
try {
spark.sql(sql)
} catch {
- case e: Throwable if getRootCause(e).getMessage.matches(errorMsgRegex) =>
+ case e: Throwable if HoodieTestUtils.getRootCause(e).getMessage.matches(errorMsgRegex) =>
hasException = true
case f: Throwable =>
- fail("Exception should match pattern: " + errorMsgRegex + ", error message: " + getRootCause(f).getMessage, f)
+ fail("Exception should match pattern: " + errorMsgRegex + ", error message: " + HoodieTestUtils.getRootCause(f).getMessage, f)
}
assertResult(true)(hasException)
}
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index c9c44fd70216..85746e3ea7f5 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -82,7 +82,7 @@ import org.apache.hudi.execution.bulkinsert.BulkInsertSortMode;
import org.apache.hudi.hadoop.fs.HadoopFSUtils;
import org.apache.hudi.hive.HiveSyncConfig;
import org.apache.hudi.hive.HoodieHiveSyncClient;
-import org.apache.hudi.io.hadoop.HoodieAvroParquetReader;
+import org.apache.hudi.io.storage.hadoop.HoodieAvroParquetReader;
import org.apache.hudi.keygen.ComplexKeyGenerator;
import org.apache.hudi.keygen.CustomKeyGenerator;
import org.apache.hudi.keygen.NonpartitionedKeyGenerator;