You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by GitBox <gi...@apache.org> on 2021/11/17 02:28:02 UTC
[GitHub] [hudi] yanghua commented on a change in pull request #3813: [HUDI-2563][hudi-client] Refactor CompactionTriggerStrategy.
yanghua commented on a change in pull request #3813:
URL: https://github.com/apache/hudi/pull/3813#discussion_r750834212
##########
File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactionTriggerStrategy.java
##########
@@ -18,13 +18,114 @@
package org.apache.hudi.table.action.compact;
+import java.text.ParseException;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieCompactionException;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+@SuppressWarnings("checkstyle:LineLength")
public enum CompactionTriggerStrategy {
- // trigger compaction when reach N delta commits
- NUM_COMMITS,
- // trigger compaction when time elapsed > N seconds since last compaction
- TIME_ELAPSED,
- // trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied
- NUM_AND_TIME,
- // trigger compaction when NUM_COMMITS or TIME_ELAPSED is satisfied
- NUM_OR_TIME
+
+ // trigger compaction when reach N delta commits
+ NUM_COMMITS {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft();
+ if (compactable) {
+ LOG.info(String.format("The delta commits >= %s, trigger compaction scheduler.", compactInlineMaxDeltaCommits));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when time elapsed > N seconds since last compaction
+ TIME_ELAPSED {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
+ if (compactable) {
+ LOG.info(String.format("The elapsed time >=%ss, trigger compaction scheduler.", compactInlineMaxDeltaSeconds));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied
+ NUM_AND_TIME {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft()
+ && compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
+ if (compactable) {
+ LOG.info(String.format("The delta commits >= %s or elapsed_time >=%ss, trigger compaction scheduler.",
+ compactInlineMaxDeltaCommits, compactInlineMaxDeltaSeconds));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when NUM_COMMITS or TIME_ELAPSED is satisfied
+ NUM_OR_TIME {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft()
+ || compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
Review comment:
ditto
##########
File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactionTriggerStrategy.java
##########
@@ -18,13 +18,114 @@
package org.apache.hudi.table.action.compact;
+import java.text.ParseException;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieCompactionException;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+@SuppressWarnings("checkstyle:LineLength")
public enum CompactionTriggerStrategy {
- // trigger compaction when reach N delta commits
- NUM_COMMITS,
- // trigger compaction when time elapsed > N seconds since last compaction
- TIME_ELAPSED,
- // trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied
- NUM_AND_TIME,
- // trigger compaction when NUM_COMMITS or TIME_ELAPSED is satisfied
- NUM_OR_TIME
+
+ // trigger compaction when reach N delta commits
+ NUM_COMMITS {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft();
+ if (compactable) {
+ LOG.info(String.format("The delta commits >= %s, trigger compaction scheduler.", compactInlineMaxDeltaCommits));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when time elapsed > N seconds since last compaction
+ TIME_ELAPSED {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
Review comment:
this style is strange. can we do not start with `<=` and put `(parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight())` into a variable?
##########
File path: hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/action/compact/CompactionTriggerStrategy.java
##########
@@ -18,13 +18,114 @@
package org.apache.hudi.table.action.compact;
+import java.text.ParseException;
+import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.exception.HoodieCompactionException;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+@SuppressWarnings("checkstyle:LineLength")
public enum CompactionTriggerStrategy {
- // trigger compaction when reach N delta commits
- NUM_COMMITS,
- // trigger compaction when time elapsed > N seconds since last compaction
- TIME_ELAPSED,
- // trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied
- NUM_AND_TIME,
- // trigger compaction when NUM_COMMITS or TIME_ELAPSED is satisfied
- NUM_OR_TIME
+
+ // trigger compaction when reach N delta commits
+ NUM_COMMITS {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft();
+ if (compactable) {
+ LOG.info(String.format("The delta commits >= %s, trigger compaction scheduler.", compactInlineMaxDeltaCommits));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when time elapsed > N seconds since last compaction
+ TIME_ELAPSED {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
+ if (compactable) {
+ LOG.info(String.format("The elapsed time >=%ss, trigger compaction scheduler.", compactInlineMaxDeltaSeconds));
+ }
+ return compactable;
+ }
+ },
+ // trigger compaction when both NUM_COMMITS and TIME_ELAPSED are satisfied
+ NUM_AND_TIME {
+ @Override
+ public boolean compactable(
+ long compactInlineMaxDeltaCommits,
+ long compactInlineMaxDeltaSeconds,
+ Pair<Integer, String> latestDeltaCommitInfo,
+ String instantTime) {
+
+ boolean compactable = compactInlineMaxDeltaCommits <= latestDeltaCommitInfo.getLeft()
+ && compactInlineMaxDeltaSeconds
+ <= (parsedToSeconds(instantTime) - parsedToSeconds(latestDeltaCommitInfo.getRight()));
Review comment:
ditto
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscribe@hudi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org