You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by vb...@apache.org on 2020/04/22 03:12:41 UTC
[incubator-hudi] branch master updated: [HUDI-821] Fixing
JCommander param parsing in deltastreamer (#1525)
This is an automated email from the ASF dual-hosted git repository.
vbalaji pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2a56f82 [HUDI-821] Fixing JCommander param parsing in deltastreamer (#1525)
2a56f82 is described below
commit 2a56f82908a8b8f788a7547d3c707c144696c1df
Author: Alexander Filipchik <af...@gmail.com>
AuthorDate: Tue Apr 21 20:12:34 2020 -0700
[HUDI-821] Fixing JCommander param parsing in deltastreamer (#1525)
Co-authored-by: Alex Filipchik <al...@csscompany.com>
---
.../utilities/deltastreamer/HoodieDeltaStreamer.java | 16 +---------------
.../deltastreamer/HoodieMultiTableDeltaStreamer.java | 3 +--
2 files changed, 2 insertions(+), 17 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
index f849b9e..3b714b6 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamer.java
@@ -54,7 +54,6 @@ import org.apache.spark.sql.SparkSession;
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
@@ -65,7 +64,6 @@ import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
-import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
@@ -164,17 +162,6 @@ public class HoodieDeltaStreamer implements Serializable {
}
}
- protected static class TransformersConverter implements IStringConverter<List<String>> {
-
- @Override
- public List<String> convert(String value) throws ParameterException {
- return value == null ? null : Arrays.stream(value.split(","))
- .map(String::trim)
- .filter(s -> !s.isEmpty())
- .collect(Collectors.toList());
- }
- }
-
public static class Config implements Serializable {
@Parameter(names = {"--target-base-path"},
@@ -228,8 +215,7 @@ public class HoodieDeltaStreamer implements Serializable {
+ ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before "
+ "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which "
+ "allows a SQL query templated to be passed as a transformation function). "
- + "Pass a comma-separated list of subclass names to chain the transformations.",
- converter = TransformersConverter.class)
+ + "Pass a comma-separated list of subclass names to chain the transformations.")
public List<String> transformerClassNames = null;
@Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. "
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java
index 836d992..58501f6 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/HoodieMultiTableDeltaStreamer.java
@@ -256,8 +256,7 @@ public class HoodieMultiTableDeltaStreamer {
+ ". Allows transforming raw source Dataset to a target Dataset (conforming to target schema) before "
+ "writing. Default : Not set. E:g - org.apache.hudi.utilities.transform.SqlQueryBasedTransformer (which "
+ "allows a SQL query templated to be passed as a transformation function). "
- + "Pass a comma-separated list of subclass names to chain the transformations.",
- converter = HoodieDeltaStreamer.TransformersConverter.class)
+ + "Pass a comma-separated list of subclass names to chain the transformations.")
public List<String> transformerClassNames = null;
@Parameter(names = {"--source-limit"}, description = "Maximum amount of data to read from source. "