You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@uniffle.apache.org by GitBox <gi...@apache.org> on 2022/08/01 04:16:55 UTC
[GitHub] [incubator-uniffle] smallzhongfeng commented on a diff in pull request #104: [Improvement] Use ConfigBuilder to rewrite the class RssSparkConfig
smallzhongfeng commented on code in PR #104:
URL: https://github.com/apache/incubator-uniffle/pull/104#discussion_r934124894
##########
client-spark/common/src/main/java/org/apache/spark/shuffle/RssSparkConfig.java:
##########
@@ -20,127 +20,268 @@
import java.util.Set;
import com.google.common.collect.Sets;
+import org.apache.spark.internal.config.ConfigBuilder;
+import org.apache.spark.internal.config.ConfigEntry;
+import org.apache.spark.internal.config.TypedConfigBuilder;
+import scala.runtime.AbstractFunction1;
import org.apache.uniffle.client.util.RssClientConfig;
+import org.apache.uniffle.common.config.ConfigUtils;
public class RssSparkConfig {
public static final String SPARK_RSS_CONFIG_PREFIX = "spark.";
- public static final String RSS_PARTITION_NUM_PER_RANGE =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_PARTITION_NUM_PER_RANGE;
- public static final int RSS_PARTITION_NUM_PER_RANGE_DEFAULT_VALUE =
- RssClientConfig.RSS_PARTITION_NUM_PER_RANGE_DEFAULT_VALUE;
- public static final String RSS_WRITER_BUFFER_SIZE =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_WRITER_BUFFER_SIZE;
- public static final String RSS_WRITER_BUFFER_SIZE_DEFAULT_VALUE = "3m";
- public static final String RSS_WRITER_SERIALIZER_BUFFER_SIZE =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.serializer.buffer.size";
- public static final String RSS_WRITER_SERIALIZER_BUFFER_SIZE_DEFAULT_VALUE = "3k";
- public static final String RSS_WRITER_BUFFER_SEGMENT_SIZE =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.buffer.segment.size";
- public static final String RSS_WRITER_BUFFER_SEGMENT_SIZE_DEFAULT_VALUE = "3k";
- public static final String RSS_WRITER_BUFFER_SPILL_SIZE =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.buffer.spill.size";
- public static final String RSS_WRITER_BUFFER_SPILL_SIZE_DEFAULT_VALUE = "128m";
- public static final String RSS_WRITER_PRE_ALLOCATED_BUFFER_SIZE =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.pre.allocated.buffer.size";
- public static final String RSS_WRITER_PRE_ALLOCATED_BUFFER_SIZE_DEFAULT_VALUE = "16m";
- public static final String RSS_WRITER_REQUIRE_MEMORY_RETRY_MAX =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.require.memory.retryMax";
- public static final int RSS_WRITER_REQUIRE_MEMORY_RETRY_MAX_DEFAULT_VALUE = 1200;
- public static final String RSS_WRITER_REQUIRE_MEMORY_INTERVAL =
- SPARK_RSS_CONFIG_PREFIX + "rss.writer.require.memory.interval";
- public static final long RSS_WRITER_REQUIRE_MEMORY_INTERVAL_DEFAULT_VALUE = 1000; // 1s
- public static final String RSS_COORDINATOR_QUORUM =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_COORDINATOR_QUORUM;
- public static final String RSS_CLIENT_SEND_CHECK_TIMEOUT_MS =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_SEND_CHECK_TIMEOUT_MS;
- public static final long RSS_CLIENT_SEND_CHECK_TIMEOUT_MS_DEFAULT_VALUE =
- RssClientConfig.RSS_CLIENT_SEND_CHECK_TIMEOUT_MS_DEFAULT_VALUE; // 10 min
- public static final String RSS_CLIENT_SEND_CHECK_INTERVAL_MS =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_SEND_CHECK_INTERVAL_MS;
- public static final long RSS_CLIENT_SEND_CHECK_INTERVAL_MS_DEFAULT_VALUE =
- RssClientConfig.RSS_CLIENT_SEND_CHECK_INTERVAL_MS_DEFAULT_VALUE;
- public static final String RSS_TEST_FLAG = SPARK_RSS_CONFIG_PREFIX + "rss.test";
- public static final String RSS_REMOTE_STORAGE_PATH =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_REMOTE_STORAGE_PATH;
- public static final String RSS_INDEX_READ_LIMIT =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_INDEX_READ_LIMIT;
- public static final int RSS_INDEX_READ_LIMIT_DEFAULT_VALUE = 500;
- public static final String RSS_CLIENT_TYPE =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_TYPE;
- public static final String RSS_CLIENT_TYPE_DEFAULT_VALUE = RssClientConfig.RSS_CLIENT_TYPE_DEFAULT_VALUE;
- public static final String RSS_STORAGE_TYPE = SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_STORAGE_TYPE;
- public static final String RSS_CLIENT_RETRY_MAX = SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_RETRY_MAX;
- public static final int RSS_CLIENT_RETRY_MAX_DEFAULT_VALUE = RssClientConfig.RSS_CLIENT_RETRY_MAX_DEFAULT_VALUE;
- public static final String RSS_CLIENT_RETRY_INTERVAL_MAX =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_RETRY_INTERVAL_MAX;
- public static final long RSS_CLIENT_RETRY_INTERVAL_MAX_DEFAULT_VALUE =
- RssClientConfig.RSS_CLIENT_RETRY_INTERVAL_MAX_DEFAULT_VALUE;
- public static final String RSS_CLIENT_HEARTBEAT_THREAD_NUM =
- SPARK_RSS_CONFIG_PREFIX + "rss.client.heartBeat.threadNum";
- public static final int RSS_CLIENT_HEARTBEAT_THREAD_NUM_DEFAULT_VALUE = 4;
- public static final String RSS_CLIENT_SEND_SIZE_LIMIT = SPARK_RSS_CONFIG_PREFIX + "rss.client.send.size.limit";
- public static final String RSS_CLIENT_SEND_SIZE_LIMIT_DEFAULT_VALUE = "16m";
- public static final String RSS_CLIENT_READ_BUFFER_SIZE =
- SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_CLIENT_READ_BUFFER_SIZE;
+
+ public static final ConfigEntry<Integer> RSS_PARTITION_NUM_PER_RANGE = createIntegerBuilder(
+ new ConfigBuilder(SPARK_RSS_CONFIG_PREFIX + RssClientConfig.RSS_PARTITION_NUM_PER_RANGE)
+ .doc(""))
Review Comment:
done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@uniffle.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@uniffle.apache.org
For additional commands, e-mail: issues-help@uniffle.apache.org