You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2017/02/23 22:47:59 UTC
ambari git commit: AMBARI-20056 : FIFO compaction resulting in too
many store files on large cluster. (avijayan)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 a65bf3612 -> 016c7ac48
AMBARI-20056 : FIFO compaction resulting in too many store files on large cluster. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/016c7ac4
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/016c7ac4
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/016c7ac4
Branch: refs/heads/branch-2.5
Commit: 016c7ac48dff60e4506f038ca053cb73c8962337
Parents: a65bf36
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Thu Feb 23 14:47:52 2017 -0800
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Thu Feb 23 14:47:52 2017 -0800
----------------------------------------------------------------------
.../timeline/HBaseTimelineMetricStore.java | 18 +-
.../metrics/timeline/PhoenixHBaseAccessor.java | 199 +++++++++++--------
.../timeline/TimelineMetricConfiguration.java | 25 ++-
.../timeline/ITPhoenixHBaseAccessor.java | 21 +-
.../server/upgrade/UpgradeCatalog250.java | 7 +
.../0.1.0/configuration/ams-site.xml | 8 -
.../server/upgrade/UpgradeCatalog250Test.java | 20 +-
7 files changed, 189 insertions(+), 109 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
index fdd1057..72ae4ac 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java
@@ -106,13 +106,17 @@ public class HBaseTimelineMetricStore extends AbstractService implements Timelin
hBaseAccessor.initPoliciesAndTTL();
// Start HA service
// Start the controller
- haController = new MetricCollectorHAController(configuration);
- try {
- haController.initializeHAController();
- } catch (Exception e) {
- LOG.error(e);
- throw new MetricsSystemInitializationException("Unable to " +
- "initialize HA controller", e);
+ if (!configuration.isDistributedCollectorModeDisabled()) {
+ haController = new MetricCollectorHAController(configuration);
+ try {
+ haController.initializeHAController();
+ } catch (Exception e) {
+ LOG.error(e);
+ throw new MetricsSystemInitializationException("Unable to " +
+ "initialize HA controller", e);
+ }
+ } else {
+ LOG.info("Distributed collector mode disabled");
}
//Initialize whitelisting & blacklisting if needed
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
index ad05025..a4539c4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline;
+import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -80,6 +81,11 @@ import java.util.concurrent.TimeUnit;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATE_TABLE_SPLIT_POINTS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
@@ -103,6 +109,7 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_ENABLED;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL;
@@ -174,10 +181,12 @@ public class PhoenixHBaseAccessor {
static final String HSTORE_COMPACTION_CLASS_KEY =
"hbase.hstore.defaultengine.compactionpolicy.class";
+ static final String HSTORE_ENGINE_CLASS =
+ "hbase.hstore.engine.class";
static final String FIFO_COMPACTION_POLICY_CLASS =
"org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
- static final String DEFAULT_COMPACTION_POLICY_CLASS =
- "org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy";
+ static final String DATE_TIERED_COMPACTION_POLICY =
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
static final String BLOCKING_STORE_FILES_KEY =
"hbase.hstore.blockingStoreFiles";
@@ -491,8 +500,6 @@ public class PhoenixHBaseAccessor {
}
protected void initPoliciesAndTTL() {
- boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
- boolean enableFifoCompaction = metricsConf.getBoolean("timeline.metrics.hbase.fifo.compaction.enabled", true);
HBaseAdmin hBaseAdmin = null;
try {
@@ -507,90 +514,23 @@ public class PhoenixHBaseAccessor {
boolean modifyTable = false;
HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
+ //Set normalizer preferences
+ boolean enableNormalizer = hbaseConf.getBoolean("hbase.normalizer.enabled", false);
if (enableNormalizer ^ tableDescriptor.isNormalizationEnabled()) {
tableDescriptor.setNormalizationEnabled(enableNormalizer);
LOG.info("Normalizer set to " + enableNormalizer + " for " + tableName);
modifyTable = true;
}
- if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
- if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
- LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName);
- boolean validDurability = true;
- if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.SKIP_WAL);
- } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.SYNC_WAL);
- } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.ASYNC_WAL);
- } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
- tableDescriptor.setDurability(Durability.FSYNC_WAL);
- } else {
- LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability);
- validDurability = false;
- }
- if (validDurability) {
- modifyTable = true;
- }
- }
- } else {
- if (!timelineMetricsTablesDurability.isEmpty()) {
- LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName);
- boolean validDurability = true;
- if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.SKIP_WAL);
- } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.SYNC_WAL);
- } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.ASYNC_WAL);
- } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
- tableDescriptor.setDurability(Durability.FSYNC_WAL);
- } else {
- LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability);
- validDurability = false;
- }
- if (validDurability) {
- modifyTable = true;
- }
- }
- }
+ //Set durability preferences
+ boolean durabilitySettingsModified = setDurabilityForTable(tableName, tableDescriptor);
+ modifyTable = modifyTable || durabilitySettingsModified;
+
+ //Set compaction policy preferences
+ boolean compactionPolicyModified = false;
+ compactionPolicyModified = setCompactionPolicyForTable(tableName, tableDescriptor);
+ modifyTable = modifyTable || compactionPolicyModified;
- Map<String, String> config = tableDescriptor.getConfiguration();
- if (enableFifoCompaction &&
- !FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY))) {
- tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY,
- FIFO_COMPACTION_POLICY_CLASS);
- LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
- " = " + FIFO_COMPACTION_POLICY_CLASS + " for " + tableName);
- // Need to set blockingStoreFiles to 1000 for FIFO
- int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 1000);
- if (blockingStoreFiles < 1000) {
- blockingStoreFiles = 1000;
- }
- tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
- LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + blockingStoreFiles + " for " + tableName);
- modifyTable = true;
- }
- // Set back original policy if fifo disabled
- if (!enableFifoCompaction &&
- FIFO_COMPACTION_POLICY_CLASS.equals(config.get(HSTORE_COMPACTION_CLASS_KEY))) {
- tableDescriptor.setConfiguration(HSTORE_COMPACTION_CLASS_KEY,
- DEFAULT_COMPACTION_POLICY_CLASS);
- LOG.info("Setting config property " + HSTORE_COMPACTION_CLASS_KEY +
- " = " + DEFAULT_COMPACTION_POLICY_CLASS + " for " + tableName);
-
- int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, 300);
- if (blockingStoreFiles > 300) {
- LOG.warn("HBase blocking store file set too high without FIFO " +
- "Compaction policy enabled, restoring low value = 300.");
- blockingStoreFiles = 300;
- }
- tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(blockingStoreFiles));
- LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + blockingStoreFiles + " for " + tableName);
- modifyTable = true;
- }
// Change TTL setting to match user configuration
HColumnDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
if (columnFamilies != null) {
@@ -623,6 +563,103 @@ public class PhoenixHBaseAccessor {
}
}
+ private boolean setDurabilityForTable(String tableName, HTableDescriptor tableDescriptor) {
+
+ boolean modifyTable = false;
+ //Set WAL preferences
+ if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
+ if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
+ }
+ } else {
+ if (!timelineMetricsTablesDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
+ }
+ }
+ return modifyTable;
+ }
+
+ private boolean setCompactionPolicyForTable(String tableName, HTableDescriptor tableDescriptor) {
+
+ String compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_ENGINE_CLASS);
+ String compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS,
+ DATE_TIERED_COMPACTION_POLICY);
+ int blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES, 60);
+
+ if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+ compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_COMPACTION_CLASS_KEY);
+ compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS,
+ FIFO_COMPACTION_POLICY_CLASS);
+ blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES, 1000);
+ }
+
+ Map<String, String> config = new HashMap(tableDescriptor.getConfiguration());
+
+ if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass)) {
+ config.remove(HSTORE_COMPACTION_CLASS_KEY);
+ config.remove(HSTORE_ENGINE_CLASS);
+ //Default blockingStoreFiles = 300
+ setHbaseBlockingStoreFiles(tableDescriptor, tableName, 300);
+ } else {
+ tableDescriptor.setConfiguration(compactionPolicyKey, compactionPolicyClass);
+ setHbaseBlockingStoreFiles(tableDescriptor, tableName, blockingStoreFiles);
+ }
+
+ if (!compactionPolicyKey.equals(HSTORE_ENGINE_CLASS)) {
+ tableDescriptor.removeConfiguration(HSTORE_ENGINE_CLASS);
+ }
+ if (!compactionPolicyKey.equals(HSTORE_COMPACTION_CLASS_KEY)) {
+ tableDescriptor.removeConfiguration(HSTORE_COMPACTION_CLASS_KEY);
+ }
+
+ Map<String, String> newConfig = tableDescriptor.getConfiguration();
+ return !Maps.difference(config, newConfig).areEqual();
+ }
+
+ private void setHbaseBlockingStoreFiles(HTableDescriptor tableDescriptor, String tableName, int value) {
+ int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, value);
+ if (blockingStoreFiles != value) {
+ blockingStoreFiles = value;
+ }
+ tableDescriptor.setConfiguration(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
+ LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
+ " = " + blockingStoreFiles + " for " + tableName);
+ }
+
protected String getSplitPointsStr(String splitPoints) {
if (StringUtils.isEmpty(splitPoints.trim())) {
return "";
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
index 9c77f49..cc751c0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java
@@ -266,6 +266,24 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRIC_METADATA_FILTERS =
"timeline.metrics.service.metadata.filters";
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.precision.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.precision.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+
public static final String HOST_APP_ID = "HOST";
public static final String DEFAULT_INSTANCE_PORT = "12001";
@@ -422,9 +440,12 @@ public class TimelineMetricConfiguration {
return defaultRpcAddress;
}
- public boolean isDistributedOperationModeEnabled() {
+ public boolean isDistributedCollectorModeDisabled() {
try {
- return getMetricsConf().get("timeline.metrics.service.operation.mode").equals("distributed");
+ if (metricsConf != null) {
+ return Boolean.parseBoolean(metricsConf.get("timeline.metrics.service.distributed.collector.mode.disabled", "false"));
+ }
+ return false;
} catch (Exception e) {
return false;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
index 3bc1929..0087fd9 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java
@@ -62,8 +62,10 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createEmptyTimelineMetric;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.createMetricHostAggregate;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.MetricTestHelper.prepareSingleTimelineMetric;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY;
+import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
@@ -327,8 +329,6 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
tableDescriptor.setNormalizationEnabled(true);
Assert.assertTrue("Normalizer enabled.", tableDescriptor.isNormalizationEnabled());
- Assert.assertNull("Default compaction policy is null.",
- tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY));
for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) {
if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
@@ -352,7 +352,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
// Verify expected policies are set
boolean normalizerEnabled = false;
- String compactionPolicy = null;
+ String precisionTableCompactionPolicy = null;
+ String aggregateTableCompactionPolicy = null;
boolean tableDurabilitySet = false;
for (int i = 0; i < 10; i++) {
LOG.warn("Policy check retry : " + i);
@@ -360,12 +361,15 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
HTableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableName.getBytes());
normalizerEnabled = tableDescriptor.isNormalizationEnabled();
tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
- compactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY);
+ if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
+ precisionTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_ENGINE_CLASS);
+ } else {
+ aggregateTableCompactionPolicy = tableDescriptor.getConfigurationValue(HSTORE_COMPACTION_CLASS_KEY);
+ }
LOG.debug("Table: " + tableName + ", normalizerEnabled = " + normalizerEnabled);
- LOG.debug("Table: " + tableName + ", compactionPolicy = " + compactionPolicy);
// Best effort for 20 seconds
- if (normalizerEnabled || compactionPolicy == null) {
- Thread.sleep(2000l);
+ if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy ==null)) {
+ Thread.sleep(20000l);
}
if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
for (HColumnDescriptor family : tableDescriptor.getColumnFamilies()) {
@@ -377,7 +381,8 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
Assert.assertTrue("Durability Set.", tableDurabilitySet);
- Assert.assertEquals("FIFO compaction policy is set.", FIFO_COMPACTION_POLICY_CLASS, compactionPolicy);
+ Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
+ Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy);
Assert.assertEquals("Precision TTL value not changed.", String.valueOf(2 * 86400), precisionTtl);
hBaseAdmin.close();
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
index 6a3e751..c488cb6 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog250.java
@@ -325,6 +325,13 @@ public class UpgradeCatalog250 extends AbstractUpgradeCatalog {
if ("distributed".equals(amsSite.getProperties().get(AMS_MODE))) {
isDistributed = true;
}
+
+ Map<String, String> amsSiteProperties = amsSite.getProperties();
+
+ if (amsSiteProperties != null && amsSiteProperties.containsKey("timeline.metrics.hbase.fifo.compaction.enabled")) {
+ LOG.info("Removing timeline.metrics.hbase.fifo.compaction.enabled from ams-site");
+ removeConfigurationPropertiesFromCluster(cluster, AMS_SITE, Collections.singleton("timeline.metrics.hbase.fifo.compaction.enabled"));
+ }
}
if (isDistributed) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index 76f8660..b9f534e 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -589,14 +589,6 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
- <name>timeline.metrics.hbase.fifo.compaction.enabled</name>
- <value>true</value>
- <description>
- Enable Compaction policy for lower for Precision and Minute aggregate tables.
- </description>
- <on-ambari-upgrade add="true"/>
- </property>
- <property>
<name>timeline.metrics.aggregators.skip.blockcache.enabled</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/016c7ac4/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
index 9e36e20..3f934d7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog250Test.java
@@ -52,6 +52,7 @@ import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.stack.OsFamily;
import org.easymock.Capture;
+import org.easymock.CaptureType;
import org.easymock.EasyMock;
import org.easymock.EasyMockRunner;
import org.easymock.EasyMockSupport;
@@ -642,8 +643,16 @@ public class UpgradeCatalog250Test {
Map<String, String> amsSite = new HashMap<String, String>() {
{
put("timeline.metrics.service.operation.mode", "distributed");
+ put("timeline.metrics.hbase.fifo.compaction.enabled", "true");
}
};
+
+ Map<String, String> newAmsSite = new HashMap<String, String>() {
+ {
+ put("timeline.metrics.service.operation.mode", "distributed");
+ }
+ };
+
EasyMockSupport easyMockSupport = new EasyMockSupport();
Config mockAmsHbaseSite = easyMockSupport.createNiceMock(Config.class);
@@ -669,18 +678,23 @@ public class UpgradeCatalog250Test {
.createNiceMock();
Injector injector2 = easyMockSupport.createNiceMock(Injector.class);
- Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture();
+ Capture<Map<String, String>> propertiesCapture = EasyMock.newCapture(CaptureType.ALL);
expect(injector2.getInstance(AmbariManagementController.class)).andReturn(controller).anyTimes();
expect(controller.getClusters()).andReturn(clusters).anyTimes();
expect(controller.createConfig(anyObject(Cluster.class), anyString(), capture(propertiesCapture), anyString(),
- EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).once();
+ EasyMock.<Map<String, Map<String, String>>>anyObject())).andReturn(config).times(2);
replay(controller, injector2);
new UpgradeCatalog250(injector2).updateAMSConfigs();
easyMockSupport.verifyAll();
- Map<String, String> updatedProperties = propertiesCapture.getValue();
+ assertTrue(propertiesCapture.getValues().size() == 2);
+
+ Map<String, String> updatedProperties = propertiesCapture.getValues().get(0);
+ assertTrue(Maps.difference(newAmsSite, updatedProperties).areEqual());
+
+ updatedProperties = propertiesCapture.getValues().get(1);
assertTrue(Maps.difference(newProperties, updatedProperties).areEqual());
}