You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2019/03/04 21:03:49 UTC
[accumulo] branch master updated: Change max walogs from table to
tserver property. (#1008)
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push:
new 8b6aaa5 Change max walogs from table to tserver property. (#1008)
8b6aaa5 is described below
commit 8b6aaa57126ed1f1fe6e8485c7bd559ff64f9b54
Author: Keith Turner <kt...@apache.org>
AuthorDate: Mon Mar 4 16:03:44 2019 -0500
Change max walogs from table to tserver property. (#1008)
After the changes in #860 having a tablet property for max write ahead
logs no longer makes sense. This should be a tserver property because
the behavior controls the max walogs per tserver. This commit replaces
the table property with a tablet server property.
---
.../apache/accumulo/core/conf/AccumuloConfiguration.java | 13 +++++++++++++
.../main/java/org/apache/accumulo/core/conf/Property.java | 9 +++++++--
.../java/org/apache/accumulo/tserver/tablet/Tablet.java | 4 +++-
.../accumulo/test/performance/RollWALPerformanceIT.java | 2 +-
4 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
index 64a05a9..ec6cb0d 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/AccumuloConfiguration.java
@@ -89,6 +89,19 @@ public abstract class AccumuloConfiguration implements Iterable<Entry<String,Str
public abstract String get(Property property);
/**
+ * Given a property and a deprecated property determine which one to use base on which one is set.
+ */
+ public Property resolve(Property property, Property deprecatedProperty) {
+ if (isPropertySet(property, true)) {
+ return property;
+ } else if (isPropertySet(deprecatedProperty, true)) {
+ return deprecatedProperty;
+ } else {
+ return property;
+ }
+ }
+
+ /**
* Returns property key/value pairs in this configuration. The pairs include those defined in this
* configuration which pass the given filter, and those supplied from the parent configuration
* which are not included from here.
diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
index 9e6029b..9529b1a 100644
--- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java
+++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java
@@ -314,6 +314,10 @@ public enum Property {
+ " When there are more RFiles than this setting multiple passes must be"
+ " made, which is slower. However opening too many RFiles at once can cause"
+ " problems."),
+ TSERV_WALOG_MAX_REFERENCED("tserver.walog.max.referenced", "3", PropertyType.COUNT,
+ "When a tablet server has more than this many write ahead logs, any tablet referencing older "
+ + "logs over this threshold is minor compacted. Also any tablet referencing this many "
+ + "logs or more will be compacted."),
TSERV_WALOG_MAX_SIZE("tserver.walog.max.size", "1g", PropertyType.BYTES,
"The maximum size for each write-ahead log. See comment for property"
+ " tserver.memory.maps.max"),
@@ -599,9 +603,10 @@ public enum Property {
"A tablet is split when the combined size of RFiles exceeds this amount."),
TABLE_MAX_END_ROW_SIZE("table.split.endrow.size.max", "10K", PropertyType.BYTES,
"Maximum size of end row"),
+ @Deprecated
+ @ReplacedBy(property = Property.TSERV_WALOG_MAX_REFERENCED)
TABLE_MINC_LOGS_MAX("table.compaction.minor.logs.threshold", "3", PropertyType.COUNT,
- "When there are more than this many write-ahead logs against a tablet, it"
- + " will be minor compacted. See comment for property" + " tserver.memory.maps.max"),
+ "This property is deprecated since 2.0.0."),
TABLE_MINC_COMPACT_IDLETIME("table.compaction.minor.idle", "5m", PropertyType.TIMEDURATION,
"After a tablet has been idle (no mutations) for this time period it may have its "
+ "in-memory map flushed to disk in a minor compaction. There is no guarantee an idle "
diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
index d8b10e2..780f78c 100644
--- a/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
+++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/tablet/Tablet.java
@@ -2463,7 +2463,9 @@ public class Tablet {
public void checkIfMinorCompactionNeededForLogs(List<DfsLogger> closedLogs) {
// grab this outside of tablet lock.
- int maxLogs = tableConfiguration.getCount(Property.TABLE_MINC_LOGS_MAX);
+ @SuppressWarnings("deprecation")
+ int maxLogs = tableConfiguration.getCount(tableConfiguration
+ .resolve(Property.TSERV_WALOG_MAX_REFERENCED, Property.TABLE_MINC_LOGS_MAX));
String reason = null;
synchronized (this) {
diff --git a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
index ab78f28..f1981cb 100644
--- a/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
+++ b/test/src/main/java/org/apache/accumulo/test/performance/RollWALPerformanceIT.java
@@ -50,7 +50,7 @@ public class RollWALPerformanceIT extends ConfigurableMacBase {
protected void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) {
cfg.setProperty(Property.TSERV_WAL_REPLICATION, "1");
cfg.setProperty(Property.TSERV_WALOG_MAX_SIZE, "5M");
- cfg.setProperty(Property.TABLE_MINC_LOGS_MAX, "100");
+ cfg.setProperty(Property.TSERV_WALOG_MAX_REFERENCED, "100");
cfg.setProperty(Property.GC_CYCLE_START, "1s");
cfg.setProperty(Property.GC_CYCLE_DELAY, "1s");
cfg.useMiniDFS(true);