You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by GitBox <gi...@apache.org> on 2021/05/23 15:28:18 UTC
[GitHub] [hudi] vinothchandar commented on a change in pull request #2833: [HUDI-89] Add configOption & refactor Hudi configuration framework
vinothchandar commented on a change in pull request #2833:
URL: https://github.com/apache/hudi/pull/2833#discussion_r637562713
##########
File path: hudi-common/src/main/java/org/apache/hudi/common/config/HoodieMetadataConfig.java
##########
@@ -33,53 +33,82 @@
public static final String METADATA_PREFIX = "hoodie.metadata";
// Enable the internal Metadata Table which saves file listings
- public static final String METADATA_ENABLE_PROP = METADATA_PREFIX + ".enable";
- public static final boolean DEFAULT_METADATA_ENABLE = false;
+ public static final ConfigOption<Boolean> METADATA_ENABLE_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".enable")
+ .defaultValue(false)
+ .withDescription("Enable the internal Metadata Table which stores table level metadata such as file listings");
// Validate contents of Metadata Table on each access against the actual filesystem
- public static final String METADATA_VALIDATE_PROP = METADATA_PREFIX + ".validate";
- public static final boolean DEFAULT_METADATA_VALIDATE = false;
+ public static final ConfigOption<Boolean> METADATA_VALIDATE_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".validate")
+ .defaultValue(false)
+ .withDescription("Validate contents of Metadata Table on each access against the actual listings from DFS");
+
public static final boolean DEFAULT_METADATA_ENABLE_FOR_READERS = false;
// Enable metrics for internal Metadata Table
- public static final String METADATA_METRICS_ENABLE_PROP = METADATA_PREFIX + ".metrics.enable";
- public static final boolean DEFAULT_METADATA_METRICS_ENABLE = false;
+ public static final ConfigOption<Boolean> METADATA_METRICS_ENABLE_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".metrics.enable")
+ .defaultValue(false)
+ .withDescription("");
// Parallelism for inserts
- public static final String METADATA_INSERT_PARALLELISM_PROP = METADATA_PREFIX + ".insert.parallelism";
- public static final int DEFAULT_METADATA_INSERT_PARALLELISM = 1;
+ public static final ConfigOption<Integer> METADATA_INSERT_PARALLELISM_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".insert.parallelism")
+ .defaultValue(1)
+ .withDescription("Parallelism to use when writing to the metadata table");
// Async clean
- public static final String METADATA_ASYNC_CLEAN_PROP = METADATA_PREFIX + ".clean.async";
- public static final boolean DEFAULT_METADATA_ASYNC_CLEAN = false;
+ public static final ConfigOption<Boolean> METADATA_ASYNC_CLEAN_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".clean.async")
+ .defaultValue(false)
+ .withDescription("");
// Maximum delta commits before compaction occurs
- public static final String METADATA_COMPACT_NUM_DELTA_COMMITS_PROP = METADATA_PREFIX + ".compact.max.delta.commits";
- public static final int DEFAULT_METADATA_COMPACT_NUM_DELTA_COMMITS = 24;
+ public static final ConfigOption<Integer> METADATA_COMPACT_NUM_DELTA_COMMITS_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".compact.max.delta.commits")
+ .defaultValue(24)
+ .withDescription("Controls how often the metadata table is compacted.");
// Archival settings
- public static final String MIN_COMMITS_TO_KEEP_PROP = METADATA_PREFIX + ".keep.min.commits";
- public static final int DEFAULT_MIN_COMMITS_TO_KEEP = 20;
- public static final String MAX_COMMITS_TO_KEEP_PROP = METADATA_PREFIX + ".keep.max.commits";
- public static final int DEFAULT_MAX_COMMITS_TO_KEEP = 30;
+ public static final ConfigOption<Integer> MIN_COMMITS_TO_KEEP_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".keep.min.commits")
+ .defaultValue(20)
+ .withDescription("Controls the archival of the metadata table’s timeline");
+
+ public static final ConfigOption<Integer> MAX_COMMITS_TO_KEEP_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".keep.max.commits")
+ .defaultValue(30)
+ .withDescription("Controls the archival of the metadata table’s timeline");
// Cleaner commits retained
- public static final String CLEANER_COMMITS_RETAINED_PROP = METADATA_PREFIX + ".cleaner.commits.retained";
- public static final int DEFAULT_CLEANER_COMMITS_RETAINED = 3;
+ public static final ConfigOption<Integer> CLEANER_COMMITS_RETAINED_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".cleaner.commits.retained")
+ .defaultValue(3)
+ .withDescription("");
// Controls whether or not, upon failure to fetch from metadata table, should fallback to listing.
- public static final String ENABLE_FALLBACK_PROP = METADATA_PREFIX + ".fallback.enable";
- public static final String DEFAULT_ENABLE_FALLBACK = "true";
+ public static final ConfigOption<String> ENABLE_FALLBACK_PROP = ConfigOption
+ .key(METADATA_PREFIX + ".fallback.enable")
+ .defaultValue("true")
+ .withDescription("Fallback to listing from DFS, if there are any errors in fetching from metadata table");
// Regex to filter out matching directories during bootstrap
- public static final String DIRECTORY_FILTER_REGEX = METADATA_PREFIX + ".dir.filter.regex";
- public static final String DEFAULT_DIRECTORY_FILTER_REGEX = "";
-
- public static final String HOODIE_ASSUME_DATE_PARTITIONING_PROP = "hoodie.assume.date.partitioning";
- public static final String DEFAULT_ASSUME_DATE_PARTITIONING = "false";
-
- public static final String FILE_LISTING_PARALLELISM_PROP = "hoodie.file.listing.parallelism";
- public static final int DEFAULT_FILE_LISTING_PARALLELISM = 1500;
+ public static final ConfigOption<String> DIRECTORY_FILTER_REGEX = ConfigOption
+ .key(METADATA_PREFIX + ".dir.filter.regex")
+ .defaultValue("")
+ .withDescription("");
+
+ public static final ConfigOption<String> HOODIE_ASSUME_DATE_PARTITIONING_PROP = ConfigOption
+ .key("hoodie.assume.date.partitioning")
+ .defaultValue("false")
+ .withDescription("Should HoodieWriteClient assume the data is partitioned by dates, i.e three levels from base path. "
+ + "This is a stop-gap to support tables created by versions < 0.3.1. Will be removed eventually");
Review comment:
this is legacy. Like really legacy. We have not turned this on in OSS ever. cc @n3nash do you still need this param? I really love to get rid of this.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org