You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/10/06 02:41:42 UTC
[hudi] branch master updated: [HUDI-4970] Update kafka-connect readme and refactor HoodieConfig#create (#6857)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new fb4f026580 [HUDI-4970] Update kafka-connect readme and refactor HoodieConfig#create (#6857)
fb4f026580 is described below
commit fb4f02658050a74179338d4cfba07ceabe688c53
Author: Sagar Sumit <sa...@gmail.com>
AuthorDate: Thu Oct 6 08:11:35 2022 +0530
[HUDI-4970] Update kafka-connect readme and refactor HoodieConfig#create (#6857)
---
.../apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java | 6 +++---
.../org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java | 6 +++---
.../main/java/org/apache/hudi/common/config/HoodieConfig.java | 9 +--------
hudi-kafka-connect/README.md | 11 +++++++----
.../sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala | 5 +++--
5 files changed, 17 insertions(+), 20 deletions(-)
diff --git a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
index ed4c952824..ff983d44ae 100644
--- a/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
+++ b/hudi-cli/src/test/java/org/apache/hudi/cli/commands/TestUpgradeDowngradeCommand.java
@@ -164,9 +164,9 @@ public class TestUpgradeDowngradeCommand extends CLIFunctionalTestHarness {
Path propertyFile = new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
// Load the properties and verify
FSDataInputStream fsDataInputStream = metaClient.getFs().open(propertyFile);
- HoodieConfig hoodieConfig = HoodieConfig.create(fsDataInputStream);
+ HoodieConfig config = new HoodieConfig();
+ config.getProps().load(fsDataInputStream);
fsDataInputStream.close();
- assertEquals(Integer.toString(expectedVersion.versionCode()), hoodieConfig
- .getString(HoodieTableConfig.VERSION));
+ assertEquals(Integer.toString(expectedVersion.versionCode()), config.getString(HoodieTableConfig.VERSION));
}
}
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
index 39dbacabac..64ee23c35e 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/upgrade/TestUpgradeDowngrade.java
@@ -770,9 +770,9 @@ public class TestUpgradeDowngrade extends HoodieClientTestBase {
Path propertyFile = new Path(metaClient.getMetaPath() + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE);
// Load the properties and verify
FSDataInputStream fsDataInputStream = metaClient.getFs().open(propertyFile);
- HoodieConfig hoodieConfig = HoodieConfig.create(fsDataInputStream);
+ HoodieConfig config = new HoodieConfig();
+ config.getProps().load(fsDataInputStream);
fsDataInputStream.close();
- assertEquals(Integer.toString(expectedVersion.versionCode()), hoodieConfig
- .getString(HoodieTableConfig.VERSION));
+ assertEquals(Integer.toString(expectedVersion.versionCode()), config.getString(HoodieTableConfig.VERSION));
}
}
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
index 366d19fe6e..91f0671cf9 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/config/HoodieConfig.java
@@ -18,15 +18,14 @@
package org.apache.hudi.common.config;
-import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.common.util.ReflectionUtils;
import org.apache.hudi.common.util.StringUtils;
import org.apache.hudi.exception.HoodieException;
+
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
-import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Modifier;
import java.util.Arrays;
@@ -42,12 +41,6 @@ public class HoodieConfig implements Serializable {
protected static final String CONFIG_VALUES_DELIMITER = ",";
- public static HoodieConfig create(FSDataInputStream inputStream) throws IOException {
- HoodieConfig config = new HoodieConfig();
- config.props.load(inputStream);
- return config;
- }
-
protected TypedProperties props;
public HoodieConfig() {
diff --git a/hudi-kafka-connect/README.md b/hudi-kafka-connect/README.md
index 449236ea5c..a1d6f812c1 100644
--- a/hudi-kafka-connect/README.md
+++ b/hudi-kafka-connect/README.md
@@ -36,10 +36,10 @@ After installing these dependencies, follow steps based on your requirement.
### 1 - Starting the environment
-For runtime dependencies, we encourage using the confluent HDFS connector jars. We have tested our setup with
-version `10.1.0`. Either use confluent-hub to install the connector or download it
-from [here](https://tinyurl.com/yb472f79). You can install the confluent-hub command-line tool by downloading Confluent
-Platform from [here](https://tinyurl.com/s2jjby53).
+For runtime dependencies, we encourage using the confluent HDFS connector jars. We have tested our setup with version `10.1.0`
+(essentially, `hadoop-common` dependency version 2.10.1 is required which comes as part of confluent HDFS connector).
+Either use confluent-hub to install the connector or download it from [here](https://tinyurl.com/yb472f79).
+You can install the confluent-hub command-line tool by downloading Confluent Platform from [here](https://tinyurl.com/s2jjby53).
Copy the entire folder to the classpath that will be used by the Hudi Kafka Connector.
@@ -145,6 +145,9 @@ successful running of the workers.
cd $KAFKA_HOME
./bin/connect-distributed.sh $HUDI_DIR/hudi-kafka-connect/demo/connect-distributed.properties
```
+Ensure that the `plugin.path` property points to the location where all connect plugins are installed.
+For this doc, it is `/usr/local/share/kafka/plugins`. If your plugins are installed at a different location,
+then please change the above property in `$HUDI_DIR/hudi-kafka-connect/demo/connect-distributed.properties`.
### 6 - To add the Hudi Sink to the Connector (delete it if you want to re-configure)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
index 99ce78a8a1..a664a4a32c 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/TestUpgradeOrDowngradeProcedure.scala
@@ -87,10 +87,11 @@ class TestUpgradeOrDowngradeProcedure extends HoodieSparkProcedureTestBase {
val propertyFile = new Path(metaClient.getMetaPath + "/" + HoodieTableConfig.HOODIE_PROPERTIES_FILE)
// Load the properties and verify
val fsDataInputStream = metaClient.getFs.open(propertyFile)
- val hoodieConfig = HoodieConfig.create(fsDataInputStream)
+ val config = new HoodieConfig
+ config.getProps.load(fsDataInputStream)
fsDataInputStream.close()
assertResult(Integer.toString(versionCode)) {
- hoodieConfig.getString(HoodieTableConfig.VERSION)
+ config.getString(HoodieTableConfig.VERSION)
}
}
}