You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by av...@apache.org on 2018/05/11 14:06:16 UTC
[ambari] 03/03: Revert "AMBARI-23804 : Refine AMS HBase region
splitting calculation based on UUID work."
This is an automated email from the ASF dual-hosted git repository.
avijayan pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 4b8a6b2e99971ac4970612fef391d6f237ae3e03
Author: Aravindan Vijayan <av...@hortonworks.com>
AuthorDate: Fri May 11 07:03:48 2018 -0700
Revert "AMBARI-23804 : Refine AMS HBase region splitting calculation based on UUID work."
This reverts commit 8bb3bcab2bbb2a582ed0db25c86881a9e07803b7.
---
.../core/timeline/HBaseTimelineMetricsService.java | 15 +-
.../core/timeline/PhoenixHBaseAccessor.java | 226 +++++++++++--------
.../core/timeline/TimelineMetricConfiguration.java | 47 +++-
.../timeline/TimelineMetricSplitPointComputer.java | 239 ---------------------
.../core/timeline/TimelineMetricStoreWatcher.java | 6 +-
.../discovery/TimelineMetricMetadataManager.java | 130 +++--------
.../core/timeline/query/PhoenixTransactSQL.java | 20 +-
.../timeline/uuid/Murmur3HashUuidGenStrategy.java | 43 ----
.../core/timeline/uuid/TimelineMetricUuid.java | 55 -----
.../ambari/metrics/webapp/TimelineWebServices.java | 16 +-
.../resources/metrics_def/AMSSMOKETESTFAKE.dat | 1 -
.../{HBASE_MASTER.dat => MASTER_HBASE.dat} | 0
.../{HBASE_REGIONSERVER.dat => SLAVE_HBASE.dat} | 0
.../metrics_def/TIMELINE_METRIC_STORE_WATCHER.dat | 1 -
.../timeline/AbstractMiniHBaseClusterTest.java | 9 +-
.../core/timeline/ITPhoenixHBaseAccessor.java | 39 ++--
.../core/timeline/PhoenixHBaseAccessorTest.java | 4 +-
.../TimelineMetricSplitPointComputerTest.java | 138 ------------
.../timeline/TimelineMetricStoreWatcherTest.java | 8 +-
.../timeline/discovery/TestMetadataManager.java | 2 +-
...est.java => TimelineMetricUuidManagerTest.java} | 132 ++++++------
.../ambari/server/upgrade/UpgradeCatalog270.java | 19 +-
.../0.1.0/configuration/ams-site.xml | 54 +++++
.../0.1.0/package/files/service-metrics/KAFKA.txt | 0
.../AMBARI_METRICS/0.1.0/package/scripts/ams.py | 28 +--
.../AMBARI_METRICS/0.1.0/package/scripts/params.py | 1 -
.../AMBARI_METRICS/0.1.0/service_advisor.py | 22 ++
.../server/upgrade/UpgradeCatalog270Test.java | 4 -
.../AMBARI_METRICS/test_service_advisor.py | 4 +
.../2.0.6/AMBARI_METRICS/test_metrics_collector.py | 2 +-
.../test/python/stacks/2.0.6/configs/default.json | 4 +-
.../stacks/2.0.6/configs/default_ams_embedded.json | 3 +-
32 files changed, 402 insertions(+), 870 deletions(-)
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
index d09d4bb..56a28dc 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/HBaseTimelineMetricsService.java
@@ -17,6 +17,7 @@
*/
package org.apache.ambari.metrics.core.timeline;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DEFAULT_TOPN_HOSTS_LIMIT;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.USE_GROUPBY_AGGREGATOR_QUERIES;
import static org.apache.ambari.metrics.core.timeline.availability.AggregationTaskRunner.ACTUAL_AGGREGATOR_NAMES;
@@ -34,6 +35,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ThreadFactory;
@@ -109,18 +111,15 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
private synchronized void initializeSubsystem() {
if (!isInitialized) {
hBaseAccessor = new PhoenixHBaseAccessor(null);
-
- // Initialize metadata
+ // Initialize schema
+ hBaseAccessor.initMetricSchema();
+ // Initialize metadata from store
try {
metricMetadataManager = new TimelineMetricMetadataManager(hBaseAccessor);
} catch (MalformedURLException | URISyntaxException e) {
throw new ExceptionInInitializerError("Unable to initialize metadata manager");
}
metricMetadataManager.initializeMetadata();
-
- // Initialize metric schema
- hBaseAccessor.initMetricSchema();
-
// Initialize policies before TTL update
hBaseAccessor.initPoliciesAndTTL();
// Start HA service
@@ -396,10 +395,6 @@ public class HBaseTimelineMetricsService extends AbstractService implements Time
return metricsFunctions;
}
- public void putMetricsSkipCache(TimelineMetrics metrics) throws SQLException, IOException {
- hBaseAccessor.insertMetricRecordsWithMetadata(metricMetadataManager, metrics, true);
- }
-
@Override
public TimelinePutResponse putMetrics(TimelineMetrics metrics) throws SQLException, IOException {
// Error indicated by the Sql exception
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
index dec7850..040df1b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessor.java
@@ -20,7 +20,6 @@ package org.apache.ambari.metrics.core.timeline;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.ambari.metrics.core.timeline.FunctionUtils.findMetricFunctions;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.AGGREGATORS_SKIP_BLOCK_CACHE;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.BLOCKING_STORE_FILES_KEY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_DAILY_TABLE_TTL;
@@ -28,32 +27,41 @@ import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguratio
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_MINUTE_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CLUSTER_SECOND_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.CONTAINER_METRICS_TTL;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_MAX_RETRIES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RESULT_LIMIT;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.GLOBAL_RETRY_INTERVAL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HBASE_BLOCKING_STORE_FILES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HBASE_ENCODING_SCHEME;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_COMPACTION_CLASS_KEY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_ENGINE_CLASS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_DAILY_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_HOUR_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HOST_MINUTE_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.METRICS_TRANSIENT_TABLE_TTL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.PRECISION_TABLE_TTL;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_COMMIT_INTERVAL;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_ENABLED;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_CACHE_SIZE;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_DURABILITY;
+import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_AGGREGATOR_SINK_CLASS;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.ALTER_METRICS_METADATA_TABLE;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_CONTAINER_METRICS_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_TRANSIENT_METRICS_TABLE_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_INSTANCE_HOST_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_AGGREGATE_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL;
+import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_TABLE_SQL;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.DEFAULT_ENCODING;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION;
@@ -124,7 +132,6 @@ import org.apache.ambari.metrics.core.timeline.sink.ExternalSinkProvider;
import org.apache.ambari.metrics.core.timeline.source.InternalMetricsSource;
import org.apache.ambari.metrics.core.timeline.source.InternalSourceProvider;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -194,8 +201,21 @@ public class PhoenixHBaseAccessor {
private TimelineMetricsAggregatorSink aggregatorSink;
private final int cacheCommitInterval;
private final boolean skipBlockCacheForAggregatorsEnabled;
+ private final String timelineMetricsTablesDurability;
+ private final String timelineMetricsPrecisionTableDurability;
private TimelineMetricMetadataManager metadataManagerInstance;
+ static final String HSTORE_COMPACTION_CLASS_KEY =
+ "hbase.hstore.defaultengine.compactionpolicy.class";
+ static final String HSTORE_ENGINE_CLASS =
+ "hbase.hstore.engine.class";
+ static final String FIFO_COMPACTION_POLICY_CLASS =
+ "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
+ static final String DATE_TIERED_COMPACTION_POLICY =
+ "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
+ static final String BLOCKING_STORE_FILES_KEY =
+ "hbase.hstore.blockingStoreFiles";
+
private Map<String, Integer> tableTTL = new HashMap<>();
private final TimelineMetricConfiguration configuration;
@@ -239,9 +259,11 @@ public class PhoenixHBaseAccessor {
this.cacheCommitInterval = Integer.valueOf(metricsConf.get(TIMELINE_METRICS_CACHE_COMMIT_INTERVAL, "3"));
this.insertCache = new ArrayBlockingQueue<TimelineMetrics>(cacheSize);
this.skipBlockCacheForAggregatorsEnabled = metricsConf.getBoolean(AGGREGATORS_SKIP_BLOCK_CACHE, false);
+ this.timelineMetricsTablesDurability = metricsConf.get(TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY, "");
+ this.timelineMetricsPrecisionTableDurability = metricsConf.get(TIMELINE_METRICS_PRECISION_TABLE_DURABILITY, "");
tableTTL.put(METRICS_RECORD_TABLE_NAME, metricsConf.getInt(PRECISION_TABLE_TTL, 1 * 86400)); // 1 day
- tableTTL.put(CONTAINER_METRICS_TABLE_NAME, metricsConf.getInt(CONTAINER_METRICS_TTL, 14 * 86400)); // 30 days
+ tableTTL.put(CONTAINER_METRICS_TABLE_NAME, metricsConf.getInt(CONTAINER_METRICS_TTL, 30 * 86400)); // 30 days
tableTTL.put(METRICS_AGGREGATE_MINUTE_TABLE_NAME, metricsConf.getInt(HOST_MINUTE_TABLE_TTL, 7 * 86400)); //7 days
tableTTL.put(METRICS_AGGREGATE_HOURLY_TABLE_NAME, metricsConf.getInt(HOST_HOUR_TABLE_TTL, 30 * 86400)); //30 days
tableTTL.put(METRICS_AGGREGATE_DAILY_TABLE_NAME, metricsConf.getInt(HOST_DAILY_TABLE_TTL, 365 * 86400)); //1 year
@@ -448,7 +470,7 @@ public class PhoenixHBaseAccessor {
return mapper.readValue(json, metricValuesTypeRef);
}
- public Connection getConnectionRetryingOnException() throws SQLException, InterruptedException {
+ private Connection getConnectionRetryingOnException() throws SQLException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
try{
@@ -489,9 +511,6 @@ public class PhoenixHBaseAccessor {
protected void initMetricSchema() {
Connection conn = null;
Statement stmt = null;
- PreparedStatement pStmt = null;
- TimelineMetricSplitPointComputer splitPointComputer = new TimelineMetricSplitPointComputer(
- metricsConf, hbaseConf, metadataManagerInstance);
String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING);
String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION);
@@ -502,6 +521,21 @@ public class PhoenixHBaseAccessor {
conn = getConnectionRetryingOnException();
stmt = conn.createStatement();
+ // Metadata
+ String metadataSql = String.format(CREATE_METRICS_METADATA_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(metadataSql);
+ stmt.executeUpdate(ALTER_METRICS_METADATA_TABLE);
+
+ String hostedAppSql = String.format(CREATE_HOSTED_APPS_METADATA_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(hostedAppSql);
+
+ //Host Instances table
+ String hostedInstancesSql = String.format(CREATE_INSTANCE_HOST_TABLE_SQL,
+ encoding, compression);
+ stmt.executeUpdate(hostedInstancesSql);
+
// Container Metrics
stmt.executeUpdate( String.format(CREATE_CONTAINER_METRICS_TABLE_SQL,
encoding, tableTTL.get(CONTAINER_METRICS_TABLE_NAME), compression));
@@ -509,15 +543,13 @@ public class PhoenixHBaseAccessor {
// Host level
String precisionSql = String.format(CREATE_METRICS_TABLE_SQL,
encoding, tableTTL.get(METRICS_RECORD_TABLE_NAME), compression);
- pStmt = prepareCreateMetricsTableStatement(conn, precisionSql, splitPointComputer.getPrecisionSplitPoints());
- pStmt.executeUpdate();
+ stmt.executeUpdate(precisionSql);
String hostMinuteAggregrateSql = String.format(CREATE_METRICS_AGGREGATE_TABLE_SQL,
METRICS_AGGREGATE_MINUTE_TABLE_NAME, encoding,
tableTTL.get(METRICS_AGGREGATE_MINUTE_TABLE_NAME),
compression);
- pStmt = prepareCreateMetricsTableStatement(conn, hostMinuteAggregrateSql, splitPointComputer.getHostAggregateSplitPoints());
- pStmt.executeUpdate();
+ stmt.executeUpdate(hostMinuteAggregrateSql);
stmt.executeUpdate(String.format(CREATE_METRICS_AGGREGATE_TABLE_SQL,
METRICS_AGGREGATE_HOURLY_TABLE_NAME, encoding,
@@ -533,9 +565,8 @@ public class PhoenixHBaseAccessor {
METRICS_CLUSTER_AGGREGATE_TABLE_NAME, encoding,
tableTTL.get(METRICS_CLUSTER_AGGREGATE_TABLE_NAME),
compression);
- pStmt = prepareCreateMetricsTableStatement(conn, aggregateSql, splitPointComputer.getClusterAggregateSplitPoints());
- pStmt.executeUpdate();
+ stmt.executeUpdate(aggregateSql);
stmt.executeUpdate(String.format(CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL,
METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME, encoding,
tableTTL.get(METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME),
@@ -572,13 +603,6 @@ public class PhoenixHBaseAccessor {
// Ignore
}
}
- if (pStmt != null) {
- try {
- pStmt.close();
- } catch (Exception e) {
- // Ignore
- }
- }
if (conn != null) {
try {
conn.close();
@@ -589,7 +613,7 @@ public class PhoenixHBaseAccessor {
}
}
- void initPoliciesAndTTL() {
+ protected void initPoliciesAndTTL() {
Admin hBaseAdmin = null;
try {
hBaseAdmin = dataSource.getHBaseAdmin();
@@ -598,13 +622,9 @@ public class PhoenixHBaseAccessor {
}
TableName[] tableNames = null;
- TableName[] containerMetricsTableName = null;
-
if (hBaseAdmin != null) {
try {
tableNames = hBaseAdmin.listTableNames(PHOENIX_TABLES_REGEX_PATTERN, false);
- containerMetricsTableName = hBaseAdmin.listTableNames(CONTAINER_METRICS_TABLE_NAME, false);
- tableNames = (TableName[]) ArrayUtils.addAll(tableNames, containerMetricsTableName);
} catch (IOException e) {
LOG.warn("Unable to get table names from HBaseAdmin for setting policies.", e);
return;
@@ -688,44 +708,72 @@ public class PhoenixHBaseAccessor {
}
private boolean setDurabilityForTable(String tableName, TableDescriptorBuilder tableDescriptor) {
- String tableDurability = metricsConf.get("timeline.metrics." + tableName + ".durability", "");
- if (StringUtils.isNotEmpty(tableDurability)) {
- LOG.info("Setting WAL option " + tableDurability + " for table : " + tableName);
- boolean validDurability = true;
- if ("SKIP_WAL".equals(tableDurability)) {
- tableDescriptor.setDurability(Durability.SKIP_WAL);
- } else if ("SYNC_WAL".equals(tableDurability)) {
- tableDescriptor.setDurability(Durability.SYNC_WAL);
- } else if ("ASYNC_WAL".equals(tableDurability)) {
- tableDescriptor.setDurability(Durability.ASYNC_WAL);
- } else if ("FSYNC_WAL".equals(tableDurability)) {
- tableDescriptor.setDurability(Durability.FSYNC_WAL);
- } else {
- LOG.info("Unknown value for durability : " + tableDurability);
- validDurability = false;
+
+ boolean modifyTable = false;
+
+ if (METRIC_TRANSIENT_TABLE_NAME.equalsIgnoreCase(tableName)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ modifyTable = true;
+ } else if (METRICS_RECORD_TABLE_NAME.equals(tableName)) {
+ if (!timelineMetricsPrecisionTableDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsPrecisionTableDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsPrecisionTableDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TIMELINE_METRICS_PRECISION_TABLE_DURABILITY + " : " + timelineMetricsPrecisionTableDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
+ }
+ } else {
+ if (!timelineMetricsTablesDurability.isEmpty()) {
+ LOG.info("Setting WAL option " + timelineMetricsTablesDurability + " for table : " + tableName);
+ boolean validDurability = true;
+ if ("SKIP_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SKIP_WAL);
+ } else if ("SYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.SYNC_WAL);
+ } else if ("ASYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.ASYNC_WAL);
+ } else if ("FSYNC_WAL".equals(timelineMetricsTablesDurability)) {
+ tableDescriptor.setDurability(Durability.FSYNC_WAL);
+ } else {
+ LOG.info("Unknown value for " + TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY + " : " + timelineMetricsTablesDurability);
+ validDurability = false;
+ }
+ if (validDurability) {
+ modifyTable = true;
+ }
}
- return validDurability;
}
- return false;
+ return modifyTable;
}
-
private boolean setCompactionPolicyForTable(String tableName, TableDescriptorBuilder tableDescriptorBuilder) {
boolean modifyTable = false;
- String keyConfig = "timeline.metrics." + tableName + ".compaction.policy.key";
- String policyConfig = "timeline.metrics." + tableName + ".compaction.policy";
- String storeFilesConfig = "timeline.metrics." + tableName + ".blocking.store.files";
-
- String compactionPolicyKey = metricsConf.get(keyConfig, HSTORE_ENGINE_CLASS);
- String compactionPolicyClass = metricsConf.get(policyConfig, DATE_TIERED_COMPACTION_POLICY);
- int blockingStoreFiles = hbaseConf.getInt(storeFilesConfig, 60);
-
- if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
- compactionPolicyKey = metricsConf.get(keyConfig, HSTORE_COMPACTION_CLASS_KEY);
- compactionPolicyClass = metricsConf.get(policyConfig, FIFO_COMPACTION_POLICY_CLASS);
- blockingStoreFiles = hbaseConf.getInt(storeFilesConfig, 1000);
+ String compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_ENGINE_CLASS);
+ String compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS,
+ DATE_TIERED_COMPACTION_POLICY);
+ int blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES, 60);
+
+ if (tableName.equals(METRICS_RECORD_TABLE_NAME) || tableName.equalsIgnoreCase(METRIC_TRANSIENT_TABLE_NAME)) {
+ compactionPolicyKey = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY,
+ HSTORE_COMPACTION_CLASS_KEY);
+ compactionPolicyClass = metricsConf.get(TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS,
+ FIFO_COMPACTION_POLICY_CLASS);
+ blockingStoreFiles = hbaseConf.getInt(TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES, 1000);
}
if (StringUtils.isEmpty(compactionPolicyKey) || StringUtils.isEmpty(compactionPolicyClass)) {
@@ -733,54 +781,46 @@ public class PhoenixHBaseAccessor {
modifyTable = setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, 300);
} else {
tableDescriptorBuilder.setValue(compactionPolicyKey, compactionPolicyClass);
- setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, blockingStoreFiles);
- modifyTable = true;
- }
-
- if (!compactionPolicyKey.equals(HSTORE_ENGINE_CLASS)) {
tableDescriptorBuilder.removeValue(HSTORE_ENGINE_CLASS.getBytes());
- }
- if (!compactionPolicyKey.equals(HSTORE_COMPACTION_CLASS_KEY)) {
tableDescriptorBuilder.removeValue(HSTORE_COMPACTION_CLASS_KEY.getBytes());
+ setHbaseBlockingStoreFiles(tableDescriptorBuilder, tableName, blockingStoreFiles);
+ modifyTable = true;
}
return modifyTable;
}
private boolean setHbaseBlockingStoreFiles(TableDescriptorBuilder tableDescriptor, String tableName, int value) {
- tableDescriptor.setValue(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
- LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
- " = " + value + " for " + tableName);
- return true;
- }
-
-
- private PreparedStatement prepareCreateMetricsTableStatement(Connection connection,
- String sql,
- List<byte[]> splitPoints) throws SQLException {
-
- String createTableWithSplitPointsSql = sql + getSplitPointsStr(splitPoints.size());
- LOG.info(createTableWithSplitPointsSql);
- PreparedStatement statement = connection.prepareStatement(createTableWithSplitPointsSql);
- for (int i = 1; i <= splitPoints.size(); i++) {
- statement.setBytes(i, splitPoints.get(i - 1));
+ int blockingStoreFiles = hbaseConf.getInt(HBASE_BLOCKING_STORE_FILES, value);
+ if (blockingStoreFiles != value) {
+ blockingStoreFiles = value;
+ tableDescriptor.setValue(BLOCKING_STORE_FILES_KEY, String.valueOf(value));
+ LOG.info("Setting config property " + BLOCKING_STORE_FILES_KEY +
+ " = " + blockingStoreFiles + " for " + tableName);
+ return true;
}
- return statement;
+ return false;
}
- private String getSplitPointsStr(int numSplits) {
- if (numSplits <= 0) {
+ protected String getSplitPointsStr(String splitPoints) {
+ if (StringUtils.isEmpty(splitPoints.trim())) {
return "";
}
- StringBuilder sb = new StringBuilder(" SPLIT ON ");
- sb.append("(");
- for (int i = 0; i < numSplits; i++) {
- sb.append("?");
- sb.append(",");
+ String[] points = splitPoints.split(",");
+ if (points.length > 0) {
+ StringBuilder sb = new StringBuilder(" SPLIT ON ");
+ sb.append("(");
+ for (String point : points) {
+ sb.append("'");
+ sb.append(point.trim());
+ sb.append("'");
+ sb.append(",");
+ }
+ sb.deleteCharAt(sb.length() - 1);
+ sb.append(")");
+ return sb.toString();
}
- sb.deleteCharAt(sb.length() - 1);
- sb.append(")");
- return sb.toString();
+ return "";
}
/**
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
index 393d4a3..6ec2c6b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricConfiguration.java
@@ -237,6 +237,12 @@ public class TimelineMetricConfiguration {
public static final String WATCHER_MAX_FAILURES =
"timeline.metrics.service.watcher.max.failures";
+ public static final String PRECISION_TABLE_SPLIT_POINTS =
+ "timeline.metrics.host.aggregate.splitpoints";
+
+ public static final String AGGREGATE_TABLE_SPLIT_POINTS =
+ "timeline.metrics.cluster.aggregate.splitpoints";
+
public static final String AGGREGATORS_SKIP_BLOCK_CACHE =
"timeline.metrics.aggregators.skip.blockcache.enabled";
@@ -255,6 +261,12 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD =
"timeline.metrics.sink.collection.period";
+ public static final String TIMELINE_METRICS_PRECISION_TABLE_DURABILITY =
+ "timeline.metrics.precision.table.durability";
+
+ public static final String TIMELINE_METRICS_AGGREGATE_TABLES_DURABILITY =
+ "timeline.metrics.aggregate.tables.durability";
+
public static final String TIMELINE_METRICS_WHITELIST_ENABLED =
"timeline.metrics.whitelisting.enabled";
@@ -273,9 +285,33 @@ public class TimelineMetricConfiguration {
public static final String TIMELINE_METRICS_APPS_WHITELIST =
"timeline.metrics.apps.whitelist";
+ public static final String HBASE_BLOCKING_STORE_FILES =
+ "hbase.hstore.blockingStoreFiles";
+
+ public static final String DEFAULT_TOPN_HOSTS_LIMIT =
+ "timeline.metrics.default.topn.hosts.limit";
+
public static final String TIMELINE_METRIC_AGGREGATION_SQL_FILTERS =
"timeline.metrics.cluster.aggregation.sql.filters";
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_AGGREGATE_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.aggregate.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_AGGREGATE_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.aggregate.table.hbase.hstore.blockingStoreFiles";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_KEY =
+ "timeline.metrics.hbase.precision.table.compaction.policy.key";
+
+ public static final String TIMELINE_METRICS_HBASE_PRECISION_TABLE_COMPACTION_POLICY_CLASS =
+ "timeline.metrics.hbase.precision.table.compaction.policy.class";
+
+ public static final String TIMELINE_METRICS_PRECISION_TABLE_HBASE_BLOCKING_STORE_FILES =
+ "timeline.metrics.precision.table.hbase.hstore.blockingStoreFiles";
+
public static final String TIMELINE_METRICS_SUPPORT_MULTIPLE_CLUSTERS =
"timeline.metrics.support.multiple.clusters";
@@ -310,9 +346,6 @@ public class TimelineMetricConfiguration {
public static final String TRANSIENT_METRIC_PATTERNS = "timeline.metrics.transient.metric.patterns";
- public static final String TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS = "timeline.metrics.initial.configured.master.components";
- public static final String TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS = "timeline.metrics.initial.configured.slave.components";
-
public static final String KAFKA_SERVERS = "timeline.metrics.external.sink.kafka.bootstrap.servers";
public static final String KAFKA_ACKS = "timeline.metrics.external.sink.kafka.acks";
public static final String KAFKA_RETRIES = "timeline.metrics.external.sink.kafka.bootstrap.retries";
@@ -320,13 +353,7 @@ public class TimelineMetricConfiguration {
public static final String KAFKA_LINGER_MS = "timeline.metrics.external.sink.kafka.linger.ms";
public static final String KAFKA_BUFFER_MEM = "timeline.metrics.external.sink.kafka.buffer.memory";
public static final String KAFKA_SINK_TIMEOUT_SECONDS = "timeline.metrics.external.sink.kafka.timeout.seconds";
-
- public static final String HSTORE_COMPACTION_CLASS_KEY = "hbase.hstore.defaultengine.compactionpolicy.class";
- public static final String HSTORE_ENGINE_CLASS = "hbase.hstore.engine.class";
- public static final String FIFO_COMPACTION_POLICY_CLASS = "org.apache.hadoop.hbase.regionserver.compactions.FIFOCompactionPolicy";
- public static final String DATE_TIERED_COMPACTION_POLICY = "org.apache.hadoop.hbase.regionserver.DateTieredStoreEngine";
- public static final String BLOCKING_STORE_FILES_KEY = "hbase.hstore.blockingStoreFiles";
-
+
private Configuration hbaseConf;
private Configuration metricsConf;
private Configuration metricsSslConf;
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java
deleted file mode 100644
index 89bb843..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputer.java
+++ /dev/null
@@ -1,239 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.core.timeline;
-
-import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
-import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS;
-
-public class TimelineMetricSplitPointComputer {
-
- private static final Log LOG = LogFactory.getLog(TimelineMetricSplitPointComputer.class);
- private Set<String> masterComponents = new HashSet<>();
- private Set<String> slaveComponents = new HashSet<>();
-
- private static final int MINIMUM_PRECISION_TABLE_REGIONS = 4;
- private static final int MINIMUM_AGGREGATE_TABLE_REGIONS = 2;
- private static final int OTHER_TABLE_STATIC_REGIONS = 8;
- private static final int SLAVE_EQUIDISTANT_POINTS = 50;
- private static final int MASTER_EQUIDISTANT_POINTS = 5;
-
- private List<byte[]> precisionSplitPoints = new ArrayList<>();
- private List<byte[]> aggregateSplitPoints = new ArrayList<>();
-
- public TimelineMetricSplitPointComputer(Configuration metricsConf,
- Configuration hbaseConf,
- TimelineMetricMetadataManager timelineMetricMetadataManager) {
-
- String componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "");
- if (StringUtils.isNotEmpty(componentsString)) {
- masterComponents.addAll(Arrays.asList(componentsString.split(",")));
- }
-
- componentsString = metricsConf.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "");
- if (StringUtils.isNotEmpty(componentsString)) {
- slaveComponents.addAll(Arrays.asList(componentsString.split(",")));
- }
-
- double hbaseTotalHeapsize = metricsConf.getDouble("hbase_total_heapsize", 1024*1024*1024);
- double hbaseMemstoreUpperLimit = hbaseConf.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5);
- double hbaseMemstoreFlushSize = hbaseConf.getDouble("hbase.hregion.memstore.flush.size", 134217728);
-
- computeSplitPoints(hbaseTotalHeapsize, hbaseMemstoreUpperLimit, hbaseMemstoreFlushSize, timelineMetricMetadataManager);
- }
-
-
- private void computeSplitPoints(double hbaseTotalHeapsize,
- double hbaseMemstoreUpperLimit,
- double hbaseMemstoreFlushSize,
- TimelineMetricMetadataManager timelineMetricMetadataManager) {
-
- double memstoreMaxMemory = hbaseMemstoreUpperLimit * hbaseTotalHeapsize;
- int maxInMemoryRegions = (int) ((memstoreMaxMemory / hbaseMemstoreFlushSize) - OTHER_TABLE_STATIC_REGIONS);
-
- int targetPrecisionTableRegionCount = MINIMUM_PRECISION_TABLE_REGIONS;
- int targetAggregateTableRegionCount = MINIMUM_AGGREGATE_TABLE_REGIONS;
-
- if (maxInMemoryRegions > 2) {
- targetPrecisionTableRegionCount = Math.max(4, (int)(0.70 * maxInMemoryRegions));
- targetAggregateTableRegionCount = Math.max(2, (int)(0.15 * maxInMemoryRegions));
- }
-
- List<MetricApp> metricList = new ArrayList<>();
-
- for (String component : masterComponents) {
- metricList.addAll(getSortedMetricListForSplitPoint(component, false));
- }
-
- for (String component : slaveComponents) {
- metricList.addAll(getSortedMetricListForSplitPoint(component, true));
- }
-
- int totalMetricLength = metricList.size();
-
- if (targetPrecisionTableRegionCount > 1) {
- int idx = (int) Math.ceil(totalMetricLength / targetPrecisionTableRegionCount);
- int index = idx;
- for (int i = 0; i < targetPrecisionTableRegionCount; i++) {
- if (index < totalMetricLength - 1) {
- MetricApp metricAppService = metricList.get(index);
- byte[] uuid = timelineMetricMetadataManager.getUuid(
- new TimelineClusterMetric(metricAppService.metricName, metricAppService.appId, null, -1),
- true);
- precisionSplitPoints.add(uuid);
- index += idx;
- }
- }
- }
-
- if (targetAggregateTableRegionCount > 1) {
- int idx = (int) Math.ceil(totalMetricLength / targetAggregateTableRegionCount);
- int index = idx;
- for (int i = 0; i < targetAggregateTableRegionCount; i++) {
- if (index < totalMetricLength - 1) {
- MetricApp metricAppService = metricList.get(index);
- byte[] uuid = timelineMetricMetadataManager.getUuid(
- new TimelineClusterMetric(metricAppService.metricName, metricAppService.appId, null, -1),
- true);
- aggregateSplitPoints.add(uuid);
- index += idx;
- }
- }
- }
- }
-
- private List<MetricApp> getSortedMetricListForSplitPoint(String component, boolean isSlave) {
-
- String appId = getAppId(component);
- List<MetricApp> metricList = new ArrayList<>();
-
- ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
- if (classLoader == null) {
- classLoader = getClass().getClassLoader();
- }
-
- String strLine;
- BufferedReader bufferedReader;
-
- try (InputStream inputStream = classLoader.getResourceAsStream("metrics_def/" + appId.toUpperCase() + ".dat")) {
-
- if (inputStream != null) {
- bufferedReader = new BufferedReader(new InputStreamReader(inputStream));
- LOG.info("Found split point candidate metrics for : " + appId);
-
- while ((strLine = bufferedReader.readLine()) != null) {
- metricList.add(new MetricApp(strLine.trim(), appId));
- }
- } else {
- LOG.info("Split point candidate metrics not found for : " + appId);
- }
- } catch (Exception e) {
- LOG.info("Error reading split point candidate metrics for component : " + component);
- LOG.error(e);
- }
-
- if (isSlave) {
- return getEquidistantMetrics(metricList, SLAVE_EQUIDISTANT_POINTS);
- } else {
- return getEquidistantMetrics(metricList, MASTER_EQUIDISTANT_POINTS);
- }
- }
-
- private List<MetricApp> getEquidistantMetrics(List<MetricApp> metrics, int distance) {
- List<MetricApp> selectedMetricApps = new ArrayList<>();
-
- int idx = metrics.size() / distance;
- if (idx == 0) {
- return metrics;
- }
-
- int index = idx;
- for (int i = 0; i < distance; i++) {
- selectedMetricApps.add(metrics.get(index - 1));
- index += idx;
- }
- return selectedMetricApps;
- }
-
-
- public List<byte[]> getPrecisionSplitPoints() {
- return precisionSplitPoints;
- }
-
- public List<byte[]> getClusterAggregateSplitPoints() {
- return aggregateSplitPoints;
- }
-
- public List<byte[]> getHostAggregateSplitPoints() {
- return aggregateSplitPoints;
- }
-
- private String getAppId(String component) {
-
- if (component.equalsIgnoreCase("METRICS_COLLECTOR")) {
- return "ams-hbase";
- }
-
- if (component.equalsIgnoreCase("METRICS_MONITOR")) {
- return "HOST";
- }
- return component;
- }
-}
-
-class MetricApp implements Comparable{
- String metricName;
- String appId;
-
- MetricApp(String metricName, String appId) {
- this.metricName = metricName;
- if (appId.startsWith("hbase")) {
- this.appId = "hbase";
- } else {
- this.appId = appId;
- }
- }
-
- @Override
- public int compareTo(Object o) {
- MetricApp that = (MetricApp)o;
-
- int metricCompare = metricName.compareTo(that.metricName);
- if (metricCompare != 0) {
- return metricCompare;
- }
-
- return appId.compareTo(that.appId);
- }
-}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
index ba7ce44..0ab7929 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcher.java
@@ -45,13 +45,13 @@ public class TimelineMetricStoreWatcher implements Runnable {
private static int failures = 0;
private final TimelineMetricConfiguration configuration;
- private HBaseTimelineMetricsService timelineMetricStore;
+ private TimelineMetricStore timelineMetricStore;
//used to call timelineMetricStore blocking methods with timeout
private ExecutorService executor = Executors.newSingleThreadExecutor();
- public TimelineMetricStoreWatcher(HBaseTimelineMetricsService timelineMetricStore,
+ public TimelineMetricStoreWatcher(TimelineMetricStore timelineMetricStore,
TimelineMetricConfiguration configuration) {
this.timelineMetricStore = timelineMetricStore;
this.configuration = configuration;
@@ -100,7 +100,7 @@ public class TimelineMetricStoreWatcher implements Runnable {
Callable<TimelineMetric> task = new Callable<TimelineMetric>() {
public TimelineMetric call() throws Exception {
- timelineMetricStore.putMetricsSkipCache(metrics);
+ timelineMetricStore.putMetrics(metrics);
TimelineMetrics timelineMetrics = timelineMetricStore.getTimelineMetrics(
Collections.singletonList(FAKE_METRIC_NAME), Collections.singletonList(FAKE_HOSTNAME),
FAKE_APP_ID, null, startTime - delay * 2 * 1000,
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
index 737c2ff..1ca5bc0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/discovery/TimelineMetricMetadataManager.java
@@ -20,9 +20,7 @@ package org.apache.ambari.metrics.core.timeline.discovery;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
-import java.sql.Connection;
import java.sql.SQLException;
-import java.sql.Statement;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -40,12 +38,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.ambari.metrics.core.timeline.MetricsSystemInitializationException;
import org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration;
import org.apache.ambari.metrics.core.timeline.uuid.MetricUuidGenStrategy;
import org.apache.ambari.metrics.core.timeline.uuid.MD5UuidGenStrategy;
-import org.apache.ambari.metrics.core.timeline.uuid.Murmur3HashUuidGenStrategy;
-import org.apache.ambari.metrics.core.timeline.uuid.TimelineMetricUuid;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
@@ -58,19 +53,13 @@ import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric;
import org.apache.hadoop.metrics2.sink.timeline.TimelineMetricMetadata;
import org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor;
import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
+import org.apache.ambari.metrics.core.timeline.uuid.HashBasedUuidGenStrategy;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HBASE_ENCODING_SCHEME;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TRANSIENT_METRIC_PATTERNS;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_INIT_DELAY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.METRICS_METADATA_SYNC_SCHEDULE_DELAY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRICS_UUID_GEN_STRATEGY;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_METADATA_FILTERS;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_HOSTED_APPS_METADATA_TABLE_SQL;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_INSTANCE_HOST_TABLE_SQL;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CREATE_METRICS_METADATA_TABLE_SQL;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.DEFAULT_ENCODING;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.DEFAULT_TABLE_COMPRESSION;
import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricUtils.getJavaMetricPatterns;
import static org.apache.hadoop.metrics2.sink.timeline.TimelineMetricUtils.getJavaRegexFromSqlRegex;
@@ -78,18 +67,18 @@ public class TimelineMetricMetadataManager {
private static final Log LOG = LogFactory.getLog(TimelineMetricMetadataManager.class);
// Cache all metadata on retrieval
private final Map<TimelineMetricMetadataKey, TimelineMetricMetadata> METADATA_CACHE = new ConcurrentHashMap<>();
- private final Map<TimelineMetricUuid, TimelineMetricMetadataKey> uuidKeyMap = new ConcurrentHashMap<>();
+ private final Map<String, TimelineMetricMetadataKey> uuidKeyMap = new ConcurrentHashMap<>();
// Map to lookup apps on a host
private final Map<String, TimelineMetricHostMetadata> HOSTED_APPS_MAP = new ConcurrentHashMap<>();
- private final Map<TimelineMetricUuid, String> uuidHostMap = new ConcurrentHashMap<>();
+ private final Map<String, String> uuidHostMap = new ConcurrentHashMap<>();
private final Map<String, Set<String>> INSTANCE_HOST_MAP = new ConcurrentHashMap<>();
// Sync only when needed
AtomicBoolean SYNC_HOSTED_APPS_METADATA = new AtomicBoolean(false);
AtomicBoolean SYNC_HOSTED_INSTANCES_METADATA = new AtomicBoolean(false);
- private MetricUuidGenStrategy uuidGenStrategy = new Murmur3HashUuidGenStrategy();
+ private MetricUuidGenStrategy uuidGenStrategy = new HashBasedUuidGenStrategy();
public static final int TIMELINE_METRIC_UUID_LENGTH = 16;
- public static int HOSTNAME_UUID_LENGTH = 4;
+ public static final int HOSTNAME_UUID_LENGTH = 16;
//Transient metric patterns. No UUID management and aggregation for such metrics.
private List<String> transientMetricPatterns = new ArrayList<>();
@@ -131,54 +120,7 @@ public class TimelineMetricMetadataManager {
* Initialize Metadata from the store
*/
public void initializeMetadata() {
-
- //Create metadata schema
- Connection conn = null;
- Statement stmt = null;
-
- String encoding = metricsConf.get(HBASE_ENCODING_SCHEME, DEFAULT_ENCODING);
- String compression = metricsConf.get(HBASE_COMPRESSION_SCHEME, DEFAULT_TABLE_COMPRESSION);
-
- try {
- LOG.info("Initializing metrics metadata schema...");
- conn = hBaseAccessor.getConnectionRetryingOnException();
- stmt = conn.createStatement();
-
- // Metadata
- String metadataSql = String.format(CREATE_METRICS_METADATA_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(metadataSql);
-
- String hostedAppSql = String.format(CREATE_HOSTED_APPS_METADATA_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(hostedAppSql);
-
- //Host Instances table
- String hostedInstancesSql = String.format(CREATE_INSTANCE_HOST_TABLE_SQL,
- encoding, compression);
- stmt.executeUpdate(hostedInstancesSql);
- } catch (SQLException | InterruptedException sql) {
- LOG.error("Error creating Metrics Schema in HBase using Phoenix.", sql);
- throw new MetricsSystemInitializationException(
- "Error creating Metrics Metadata Schema in HBase using Phoenix.", sql);
- } finally {
- if (stmt != null) {
- try {
- stmt.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- if (conn != null) {
- try {
- conn.close();
- } catch (SQLException e) {
- // Ignore
- }
- }
- }
-
- metricMetadataSync = new TimelineMetricMetadataSync(this);
+ metricMetadataSync = new TimelineMetricMetadataSync(this);
// Schedule the executor to sync to store
executorService.scheduleWithFixedDelay(metricMetadataSync,
metricsConf.getInt(METRICS_METADATA_SYNC_INIT_DELAY, 120), // 2 minutes
@@ -393,26 +335,14 @@ public class TimelineMetricMetadataManager {
for (TimelineMetricMetadataKey key : METADATA_CACHE.keySet()) {
TimelineMetricMetadata timelineMetricMetadata = METADATA_CACHE.get(key);
if (timelineMetricMetadata != null && timelineMetricMetadata.getUuid() != null) {
- uuidKeyMap.put(new TimelineMetricUuid(timelineMetricMetadata.getUuid()), key);
- }
- }
-
- if (!HOSTED_APPS_MAP.isEmpty()) {
- Map.Entry<String, TimelineMetricHostMetadata> entry = HOSTED_APPS_MAP.entrySet().iterator().next();
- TimelineMetricHostMetadata timelineMetricHostMetadata = entry.getValue();
- if (timelineMetricHostMetadata.getUuid() != null && timelineMetricHostMetadata.getUuid().length == 16) {
- HOSTNAME_UUID_LENGTH = 16;
- uuidGenStrategy = new MD5UuidGenStrategy();
- } else {
- HOSTNAME_UUID_LENGTH = 4;
- uuidGenStrategy = new Murmur3HashUuidGenStrategy();
+ uuidKeyMap.put(new String(timelineMetricMetadata.getUuid()), key);
}
}
for (String host : HOSTED_APPS_MAP.keySet()) {
TimelineMetricHostMetadata timelineMetricHostMetadata = HOSTED_APPS_MAP.get(host);
if (timelineMetricHostMetadata != null && timelineMetricHostMetadata.getUuid() != null) {
- uuidHostMap.put(new TimelineMetricUuid(timelineMetricHostMetadata.getUuid()), host);
+ uuidHostMap.put(new String(timelineMetricHostMetadata.getUuid()), host);
}
}
}
@@ -424,11 +354,11 @@ public class TimelineMetricMetadataManager {
*/
private MetricUuidGenStrategy getUuidStrategy(Configuration configuration) {
String strategy = configuration.get(TIMELINE_METRICS_UUID_GEN_STRATEGY, "");
- if ("md5".equalsIgnoreCase(strategy)){
- return new MD5UuidGenStrategy();
+ if ("hash".equalsIgnoreCase(strategy)) {
+ return new HashBasedUuidGenStrategy();
} else {
//Default
- return new Murmur3HashUuidGenStrategy();
+ return new MD5UuidGenStrategy();
}
}
@@ -449,13 +379,14 @@ public class TimelineMetricMetadataManager {
}
if (!createIfNotPresent) {
- LOG.debug("UUID not found for " + hostname + ", createIfNotPresent is false");
+ LOG.warn("UUID not found for " + hostname + ", createIfNotPresent is false");
return null;
}
byte[] uuid = uuidGenStrategy.computeUuid(hostname, HOSTNAME_UUID_LENGTH);
- if (uuidHostMap.containsKey(new TimelineMetricUuid(uuid))) {
- LOG.error("Duplicate key computed for " + hostname +", Collides with " + uuidHostMap.get(uuid));
+ String uuidStr = new String(uuid);
+ if (uuidHostMap.containsKey(uuidStr)) {
+ LOG.error("Duplicate key computed for " + hostname +", Collides with " + uuidHostMap.get(uuidStr));
return null;
}
@@ -464,7 +395,7 @@ public class TimelineMetricMetadataManager {
HOSTED_APPS_MAP.put(hostname, timelineMetricHostMetadata);
}
timelineMetricHostMetadata.setUuid(uuid);
- uuidHostMap.put(new TimelineMetricUuid(uuid), hostname);
+ uuidHostMap.put(uuidStr, hostname);
return uuid;
}
@@ -489,16 +420,17 @@ public class TimelineMetricMetadataManager {
}
if (!createIfNotPresent) {
- LOG.debug("UUID not found for " + key + ", createIfNotPresent is false");
+ LOG.warn("UUID not found for " + key + ", createIfNotPresent is false");
return null;
}
- byte[] uuidBytes = uuidGenStrategy.computeUuid(timelineClusterMetric, TIMELINE_METRIC_UUID_LENGTH);
+ byte[] uuid = uuidGenStrategy.computeUuid(timelineClusterMetric, TIMELINE_METRIC_UUID_LENGTH);
- TimelineMetricUuid uuid = new TimelineMetricUuid(uuidBytes);
- if (uuidKeyMap.containsKey(uuid) && !uuidKeyMap.get(uuid).equals(key)) {
- TimelineMetricMetadataKey collidingKey = uuidKeyMap.get(uuid);
- LOG.error("Duplicate key " + uuid + " computed for " + timelineClusterMetric + ", Collides with " + collidingKey);
+ String uuidStr = new String(uuid);
+ if (uuidKeyMap.containsKey(uuidStr) && !uuidKeyMap.get(uuidStr).equals(key)) {
+ TimelineMetricMetadataKey collidingKey = (TimelineMetricMetadataKey)uuidKeyMap.get(uuidStr);
+ LOG.error("Duplicate key " + Arrays.toString(uuid) + "(" + uuid + ") computed for " + timelineClusterMetric.toString()
+ + ", Collides with " + collidingKey.toString());
return null;
}
@@ -510,10 +442,10 @@ public class TimelineMetricMetadataManager {
METADATA_CACHE.put(key, timelineMetricMetadata);
}
- timelineMetricMetadata.setUuid(uuid.uuid);
+ timelineMetricMetadata.setUuid(uuid);
timelineMetricMetadata.setIsPersisted(false);
- uuidKeyMap.put(uuid, key);
- return uuid.uuid;
+ uuidKeyMap.put(uuidStr, key);
+ return uuid;
}
/**
@@ -552,14 +484,14 @@ public class TimelineMetricMetadataManager {
return metricUuid;
}
- public String getMetricNameFromUuid(byte[] uuid) {
+ public String getMetricNameFromUuid(byte[] uuid) {
byte[] metricUuid = uuid;
if (uuid.length == TIMELINE_METRIC_UUID_LENGTH + HOSTNAME_UUID_LENGTH) {
metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
}
- TimelineMetricMetadataKey key = uuidKeyMap.get(new TimelineMetricUuid(metricUuid));
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
return key != null ? key.getMetricName() : null;
}
@@ -574,11 +506,11 @@ public class TimelineMetricMetadataManager {
}
if (uuid.length == TIMELINE_METRIC_UUID_LENGTH) {
- TimelineMetricMetadataKey key = uuidKeyMap.get(new TimelineMetricUuid(uuid));
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(uuid));
return key != null ? new TimelineMetric(key.metricName, null, key.appId, key.instanceId) : null;
} else {
byte[] metricUuid = ArrayUtils.subarray(uuid, 0, TIMELINE_METRIC_UUID_LENGTH);
- TimelineMetricMetadataKey key = uuidKeyMap.get(new TimelineMetricUuid(metricUuid));
+ TimelineMetricMetadataKey key = uuidKeyMap.get(new String(metricUuid));
if (key == null) {
LOG.error("TimelineMetricMetadataKey is null for : " + Arrays.toString(uuid));
return null;
@@ -589,7 +521,7 @@ public class TimelineMetricMetadataManager {
timelineMetric.setInstanceId(key.instanceId);
byte[] hostUuid = ArrayUtils.subarray(uuid, TIMELINE_METRIC_UUID_LENGTH, HOSTNAME_UUID_LENGTH + TIMELINE_METRIC_UUID_LENGTH);
- timelineMetric.setHostName(uuidHostMap.get(new TimelineMetricUuid(hostUuid)));
+ timelineMetric.setHostName(uuidHostMap.get(new String(hostUuid)));
return timelineMetric;
}
}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
index e0cc642..f76933a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/query/PhoenixTransactSQL.java
@@ -43,11 +43,8 @@ public class PhoenixTransactSQL {
/**
* Create table to store individual metric records.
*/
-
- public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD_UUID";
-
public static final String CREATE_METRICS_TABLE_SQL = "CREATE TABLE IF NOT " +
- "EXISTS " + METRICS_RECORD_TABLE_NAME + " (UUID BINARY(20) NOT NULL, " +
+ "EXISTS METRIC_RECORD_UUID (UUID BINARY(32) NOT NULL, " +
"SERVER_TIME BIGINT NOT NULL, " +
"METRIC_SUM DOUBLE, " +
"METRIC_COUNT UNSIGNED_INT, " +
@@ -86,7 +83,7 @@ public class PhoenixTransactSQL {
public static final String CREATE_METRICS_AGGREGATE_TABLE_SQL =
"CREATE TABLE IF NOT EXISTS %s " +
- "(UUID BINARY(20) NOT NULL, " +
+ "(UUID BINARY(32) NOT NULL, " +
"SERVER_TIME BIGINT NOT NULL, " +
"METRIC_SUM DOUBLE," +
"METRIC_COUNT UNSIGNED_INT, " +
@@ -158,7 +155,7 @@ public class PhoenixTransactSQL {
public static final String CREATE_HOSTED_APPS_METADATA_TABLE_SQL =
"CREATE TABLE IF NOT EXISTS HOSTED_APPS_METADATA_UUID " +
- "(HOSTNAME VARCHAR, UUID BINARY(4), APP_IDS VARCHAR, " +
+ "(HOSTNAME VARCHAR, UUID BINARY(16), APP_IDS VARCHAR, " +
"CONSTRAINT pk PRIMARY KEY (HOSTNAME))" +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
@@ -168,6 +165,9 @@ public class PhoenixTransactSQL {
"CONSTRAINT pk PRIMARY KEY (INSTANCE_ID, HOSTNAME))" +
"DATA_BLOCK_ENCODING='%s', COMPRESSION='%s'";
+ public static final String ALTER_METRICS_METADATA_TABLE =
+ "ALTER TABLE METRICS_METADATA_UUID ADD IF NOT EXISTS IS_WHITELISTED BOOLEAN";
+
////////////////////////////////
/**
@@ -442,6 +442,8 @@ public class PhoenixTransactSQL {
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_V1_TABLE_NAME =
"METRIC_AGGREGATE_DAILY";
+ public static final String METRICS_RECORD_TABLE_NAME = "METRIC_RECORD_UUID";
+
public static final String METRICS_AGGREGATE_MINUTE_TABLE_NAME =
"METRIC_RECORD_MINUTE_UUID";
public static final String METRICS_AGGREGATE_HOURLY_TABLE_NAME =
@@ -457,7 +459,7 @@ public class PhoenixTransactSQL {
public static final String METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME =
"METRIC_AGGREGATE_DAILY_UUID";
- public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_.*");
+ public static final Pattern PHOENIX_TABLES_REGEX_PATTERN = Pattern.compile("METRIC_.*_UUID");
public static final String[] PHOENIX_TABLES = {
METRICS_RECORD_TABLE_NAME,
@@ -467,9 +469,7 @@ public class PhoenixTransactSQL {
METRICS_CLUSTER_AGGREGATE_TABLE_NAME,
METRICS_CLUSTER_AGGREGATE_MINUTE_TABLE_NAME,
METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME,
- METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME,
- METRIC_TRANSIENT_TABLE_NAME,
- CONTAINER_METRICS_TABLE_NAME
+ METRICS_CLUSTER_AGGREGATE_DAILY_TABLE_NAME
};
public static final String DEFAULT_TABLE_COMPRESSION = "SNAPPY";
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
deleted file mode 100644
index 9418aa4..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/Murmur3HashUuidGenStrategy.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.core.timeline.uuid;
-
-import com.google.common.hash.Hashing;
-import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
-import org.apache.commons.lang.StringUtils;
-
-public class Murmur3HashUuidGenStrategy implements MetricUuidGenStrategy{
-
- @Override
- public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) {
-
- String metricString = timelineClusterMetric.getMetricName() + timelineClusterMetric.getAppId();
- if (StringUtils.isNotEmpty(timelineClusterMetric.getInstanceId())) {
- metricString += timelineClusterMetric.getInstanceId();
- }
- byte[] metricBytes = metricString.getBytes();
- return Hashing.murmur3_128().hashBytes(metricBytes).asBytes();
- }
-
- @Override
- public byte[] computeUuid(String value, int maxLength) {
- byte[] valueBytes = value.getBytes();
- return Hashing.murmur3_32().hashBytes(valueBytes).asBytes();
- }
-}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuid.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuid.java
deleted file mode 100644
index 7907ff6..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuid.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.core.timeline.uuid;
-
-import java.util.Arrays;
-
-public class TimelineMetricUuid {
- public byte[] uuid;
-
- public TimelineMetricUuid(byte[] uuid) {
- this.uuid = uuid;
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(uuid);
- }
-
- @Override
- public boolean equals(Object o) {
-
- if (this == o) {
- return false;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- TimelineMetricUuid that = (TimelineMetricUuid) o;
-
- return Arrays.equals(this.uuid, that.uuid);
- }
-
- @Override
- public String toString() {
- return Arrays.toString(uuid);
- }
-}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
index 3bcbaf6..9c88b1a 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/ambari/metrics/webapp/TimelineWebServices.java
@@ -144,8 +144,8 @@ public class TimelineWebServices {
// TODO: Check ACLs for MetricEntity using the TimelineACLManager.
// TODO: Save owner of the MetricEntity.
- if (LOG.isTraceEnabled()) {
- LOG.trace("Storing metrics: " +
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing metrics: " +
TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
}
@@ -175,8 +175,8 @@ public class TimelineWebServices {
}
try {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Storing aggregated metrics: " +
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing aggregated metrics: " +
TimelineUtils.dumpTimelineRecordtoJSON(metrics, true));
}
@@ -200,8 +200,8 @@ public class TimelineWebServices {
}
try {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Storing container metrics: " + TimelineUtils
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing container metrics: " + TimelineUtils
.dumpTimelineRecordtoJSON(metrics, true));
}
@@ -250,8 +250,8 @@ public class TimelineWebServices {
) {
init(res);
try {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Request for metrics => metricNames: " + metricNames + ", " +
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Request for metrics => metricNames: " + metricNames + ", " +
"appId: " + appId + ", instanceId: " + instanceId + ", " +
"hostname: " + hostname + ", startTime: " + startTime + ", " +
"endTime: " + endTime + ", " +
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMSSMOKETESTFAKE.dat b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMSSMOKETESTFAKE.dat
deleted file mode 100644
index f5c181a..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/AMSSMOKETESTFAKE.dat
+++ /dev/null
@@ -1 +0,0 @@
-AMBARI_METRICS.SmokeTest.FakeMetric
\ No newline at end of file
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/MASTER_HBASE.dat
similarity index 100%
rename from ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_MASTER.dat
rename to ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/MASTER_HBASE.dat
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat
similarity index 100%
rename from ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HBASE_REGIONSERVER.dat
rename to ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/SLAVE_HBASE.dat
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/TIMELINE_METRIC_STORE_WATCHER.dat b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/TIMELINE_METRIC_STORE_WATCHER.dat
deleted file mode 100644
index af73d02..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/TIMELINE_METRIC_STORE_WATCHER.dat
+++ /dev/null
@@ -1 +0,0 @@
-TimelineMetricStoreWatcher.FakeMetric
\ No newline at end of file
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
index 26078cb..258054c 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/AbstractMiniHBaseClusterTest.java
@@ -18,7 +18,6 @@
package org.apache.ambari.metrics.core.timeline;
import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.OUT_OFF_BAND_DATA_TIME_ALLOWANCE;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.UPSERT_METRICS_SQL;
import static org.apache.phoenix.end2end.ParallelStatsDisabledIT.tearDownMiniCluster;
@@ -101,12 +100,8 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
// inits connection, starts mini cluster
conn = getConnection(getUrl());
- Configuration metricsConf = new Configuration();
- metricsConf.set(TimelineMetricConfiguration.HBASE_COMPRESSION_SCHEME, "NONE");
-
- metadataManager = new TimelineMetricMetadataManager(metricsConf, hdb);
- metadataManager.initializeMetadata();
hdb.initMetricSchema();
+ metadataManager = new TimelineMetricMetadataManager(new Configuration(), hdb);
hdb.setMetadataInstance(metadataManager);
}
@@ -211,8 +206,6 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest {
metricsConf.set("timeline.metrics.transient.metric.patterns", "topology%");
// Unit tests insert values into the future
metricsConf.setLong(OUT_OFF_BAND_DATA_TIME_ALLOWANCE, 600000);
- metricsConf.set("timeline.metrics." + METRICS_RECORD_TABLE_NAME + ".durability", "SKIP_WAL");
- metricsConf.set("timeline.metrics." + METRICS_CLUSTER_AGGREGATE_TABLE_NAME + ".durability", "ASYNC_WAL");
return
new PhoenixHBaseAccessor(new TimelineMetricConfiguration(new Configuration(), metricsConf),
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
index 20fbc58..65b5a1b 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/ITPhoenixHBaseAccessor.java
@@ -19,13 +19,11 @@ package org.apache.ambari.metrics.core.timeline;
import static junit.framework.Assert.assertEquals;
import static junit.framework.Assert.assertTrue;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.DATE_TIERED_COMPACTION_POLICY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.FIFO_COMPACTION_POLICY_CLASS;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_COMPACTION_CLASS_KEY;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.HSTORE_ENGINE_CLASS;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.CONTAINER_METRICS_TABLE_NAME;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.DATE_TIERED_COMPACTION_POLICY;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.FIFO_COMPACTION_POLICY_CLASS;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.HSTORE_COMPACTION_CLASS_KEY;
+import static org.apache.ambari.metrics.core.timeline.PhoenixHBaseAccessor.HSTORE_ENGINE_CLASS;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME;
-import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES;
import static org.apache.ambari.metrics.core.timeline.query.PhoenixTransactSQL.PHOENIX_TABLES_REGEX_PATTERN;
@@ -44,7 +42,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
@@ -370,30 +367,28 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
precisionValues.put(METRICS_RECORD_TABLE_NAME, precisionTtl);
f.set(hdb, precisionValues);
+ Field f2 = PhoenixHBaseAccessor.class.getDeclaredField("timelineMetricsTablesDurability");
+ f2.setAccessible(true);
+ f2.set(hdb, "ASYNC_WAL");
+
hdb.initPoliciesAndTTL();
// Verify expected policies are set
boolean normalizerEnabled = false;
String precisionTableCompactionPolicy = null;
String aggregateTableCompactionPolicy = null;
- boolean precisionTableDurabilitySet = false;
- boolean aggregateTableDurabilitySet = false;
-
- boolean isComplete = false;
-
- for (int i = 0; i < 10 && !isComplete; i++) {
+ boolean tableDurabilitySet = false;
+ for (int i = 0; i < 10; i++) {
LOG.warn("Policy check retry : " + i);
for (String tableName : PHOENIX_TABLES) {
TableName[] tableNames = hBaseAdmin.listTableNames(PHOENIX_TABLES_REGEX_PATTERN, false);
- TableName[] containerMetricsTableName = hBaseAdmin.listTableNames(CONTAINER_METRICS_TABLE_NAME, false);
- tableNames = (TableName[]) ArrayUtils.addAll(tableNames, containerMetricsTableName);
-
Optional<TableName> tableNameOptional = Arrays.stream(tableNames)
.filter(t -> tableName.equals(t.getNameAsString())).findFirst();
TableDescriptor tableDescriptor = hBaseAdmin.getTableDescriptor(tableNameOptional.get());
normalizerEnabled = tableDescriptor.isNormalizationEnabled();
+ tableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
precisionTableCompactionPolicy = tableDescriptor.getValue(HSTORE_COMPACTION_CLASS_KEY);
} else {
@@ -403,25 +398,17 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
// Best effort for 20 seconds
if (normalizerEnabled || (precisionTableCompactionPolicy == null && aggregateTableCompactionPolicy == null)) {
Thread.sleep(2000l);
- } else {
- isComplete = true;
}
if (tableName.equals(METRICS_RECORD_TABLE_NAME)) {
- precisionTableDurabilitySet = (Durability.SKIP_WAL.equals(tableDescriptor.getDurability()));
for (ColumnFamilyDescriptor family : tableDescriptor.getColumnFamilies()) {
precisionTtl = family.getTimeToLive();
}
}
-
- if (tableName.equals(METRICS_CLUSTER_AGGREGATE_TABLE_NAME)) {
- aggregateTableDurabilitySet = (Durability.ASYNC_WAL.equals(tableDescriptor.getDurability()));
- }
}
}
Assert.assertFalse("Normalizer disabled.", normalizerEnabled);
- Assert.assertTrue("METRIC_RECORD_UUID Durability Set.", precisionTableDurabilitySet);
- Assert.assertTrue("METRIC_AGGREGATE_UUID Durability Set.", aggregateTableDurabilitySet);
+ Assert.assertTrue("Durability Set.", tableDurabilitySet);
Assert.assertEquals("FIFO compaction policy is set for METRIC_RECORD_UUID.", FIFO_COMPACTION_POLICY_CLASS, precisionTableCompactionPolicy);
Assert.assertEquals("FIFO compaction policy is set for aggregate tables", DATE_TIERED_COMPACTION_POLICY, aggregateTableCompactionPolicy);
Assert.assertEquals("Precision TTL value as expected.", 86400, precisionTtl);
@@ -454,7 +441,7 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest {
metric.setExitCode(0);
List<ContainerMetric> list = Arrays.asList(metric);
hdb.insertContainerMetrics(list);
- PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS");
+ PreparedStatement stmt = conn.prepareStatement("SELECT * FROM CONTAINER_METRICS_UUID");
ResultSet set = stmt.executeQuery();
// check each filed is set properly when read back.
boolean foundRecord = false;
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
index 63ec59e..9d1b2a4 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/PhoenixHBaseAccessorTest.java
@@ -109,7 +109,7 @@ public class PhoenixHBaseAccessorTest {
mockStatic(PhoenixTransactSQL.class);
PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
- Condition condition = new DefaultCondition(Collections.singletonList(new byte[20]), metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
+ Condition condition = new DefaultCondition(Collections.singletonList(new byte[32]), metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
expect(preparedStatementMock.executeQuery()).andReturn(rsMock);
@@ -138,7 +138,7 @@ public class PhoenixHBaseAccessorTest {
mockStatic(PhoenixTransactSQL.class);
PreparedStatement preparedStatementMock = EasyMock.createNiceMock(PreparedStatement.class);
- Condition condition = new DefaultCondition(Collections.singletonList(new byte[20]), metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
+ Condition condition = new DefaultCondition(Collections.singletonList(new byte[32]), metricNames, hostnames, "appid", "instanceid", 123L, 234L, Precision.SECONDS, 10, true);
expect(PhoenixTransactSQL.prepareGetMetricsSqlStmt(null, condition)).andReturn(preparedStatementMock).once();
ResultSet rsMock = EasyMock.createNiceMock(ResultSet.class);
RuntimeException runtimeException = EasyMock.createNiceMock(RuntimeException.class);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java
deleted file mode 100644
index 4d663cc..0000000
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricSplitPointComputerTest.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ambari.metrics.core.timeline;
-
-import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
-import org.apache.ambari.metrics.core.timeline.discovery.TimelineMetricMetadataManager;
-import org.apache.hadoop.conf.Configuration;
-import org.easymock.EasyMock;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS;
-import static org.apache.ambari.metrics.core.timeline.TimelineMetricConfiguration.TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS;
-import static org.easymock.EasyMock.anyBoolean;
-import static org.easymock.EasyMock.anyObject;
-import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.replay;
-
-
-public class TimelineMetricSplitPointComputerTest {
-
- @Test
- public void testSplitPointComputationForBasicCluster() {
-
- /**
- * HBase Total heap = 1G.
- * HDFS,HBASE,YARN services deployed.
- */
- Configuration metricsConfMock = EasyMock.createMock(Configuration.class);
-
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "")).
- andReturn("METRICS_COLLECTOR,AMBARI_SERVER,NAMENODE,RESOURCEMANAGER").once();
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "")).
- andReturn("METRICS_MONITOR,DATANODE,NODEMANAGER,HBASE_REGIONSERVER").once();
- expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(1024 * 1024 * 1024.0).once();
-
- Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
- expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once();
-
- TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class), anyBoolean())).andReturn(new byte[16]);
-
- replay(metricsConfMock, hbaseConfMock, metricMetadataManagerMock);
-
- TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
- hbaseConfMock,
- metricMetadataManagerMock);
-
- Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 3);
- Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 1);
- Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 1);
- }
-
- @Test
- public void testSplitPointComputationForMediumCluster() {
-
- /**
- * HBase Total heap = 8G.
- * All services deployed.
- */
- Configuration metricsConfMock = EasyMock.createMock(Configuration.class);
-
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "")).
- andReturn("METRICS_COLLECTOR,AMBARI_SERVER,NAMENODE,RESOURCEMANAGER," +
- "NIMBUS,HIVESERVER2,HIVEMETASTORE,HBASE_MASTER,KAFKA_BROKER").once();
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "")).
- andReturn("METRICS_MONITOR,DATANODE,NODEMANAGER,HBASE_REGIONSERVER").once();
- expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(8589934592.0).once();
-
- Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
- expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(134217728.0).once();
-
- TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class), anyBoolean())).andReturn(new byte[16]);
-
- replay(metricsConfMock, hbaseConfMock, metricMetadataManagerMock);
-
- TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
- hbaseConfMock,
- metricMetadataManagerMock);
-
- Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 16);
- Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 3);
- Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 3);
- }
-
- @Test
- public void testSplitPointComputationForLargeCluster() {
-
- /**
- * HBase Total heap = 24G.
- * All services deployed.
- */
- Configuration metricsConfMock = EasyMock.createMock(Configuration.class);
-
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_MASTER_COMPONENTS, "")).
- andReturn("METRICS_COLLECTOR,AMBARI_SERVER,NAMENODE,RESOURCEMANAGER," +
- "NIMBUS,HIVESERVER2,HIVEMETASTORE,HBASE_MASTER,KAFKA_BROKER").once();
- expect(metricsConfMock.get(TIMELINE_METRIC_INITIAL_CONFIGURED_SLAVE_COMPONENTS, "")).
- andReturn("METRICS_MONITOR,DATANODE,NODEMANAGER,HBASE_REGIONSERVER").once();
- expect(metricsConfMock.getDouble("hbase_total_heapsize", 1024*1024*1024)).andReturn(24 * 1024 * 1024 * 1024.0).once();
-
- Configuration hbaseConfMock = EasyMock.createMock(Configuration.class);
- expect(hbaseConfMock.getDouble("hbase.regionserver.global.memstore.upperLimit", 0.5)).andReturn(0.5).once();
- expect(hbaseConfMock.getDouble("hbase.hregion.memstore.flush.size", 134217728)).andReturn(2 * 134217728.0).once();
-
- TimelineMetricMetadataManager metricMetadataManagerMock = EasyMock.createNiceMock(TimelineMetricMetadataManager.class);
- expect(metricMetadataManagerMock.getUuid(anyObject(TimelineClusterMetric.class), anyBoolean())).andReturn(new byte[16]);
-
- replay(metricsConfMock, hbaseConfMock, metricMetadataManagerMock);
-
- TimelineMetricSplitPointComputer timelineMetricSplitPointComputer = new TimelineMetricSplitPointComputer(metricsConfMock,
- hbaseConfMock,
- metricMetadataManagerMock);
-
- Assert.assertEquals(timelineMetricSplitPointComputer.getPrecisionSplitPoints().size(), 28);
- Assert.assertEquals(timelineMetricSplitPointComputer.getClusterAggregateSplitPoints().size(), 6);
- Assert.assertEquals(timelineMetricSplitPointComputer.getHostAggregateSplitPoints().size(), 6);
- }
-}
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
index eb64198..de0236c 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/TimelineMetricStoreWatcherTest.java
@@ -48,10 +48,10 @@ public class TimelineMetricStoreWatcherTest {
@Test
public void testRunPositive() throws Exception {
- HBaseTimelineMetricsService metricStore = createNiceMock(HBaseTimelineMetricsService.class);
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
- metricStore.putMetricsSkipCache(anyObject(TimelineMetrics.class));
- expectLastCall().once();
+ expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
+ .andReturn(new TimelinePutResponse());
// metric found
expect(metricStore.getTimelineMetrics(EasyMock.<List<String>>anyObject(),
@@ -75,7 +75,7 @@ public class TimelineMetricStoreWatcherTest {
@Test
public void testRunNegative() throws Exception {
- HBaseTimelineMetricsService metricStore = createNiceMock(HBaseTimelineMetricsService.class);
+ TimelineMetricStore metricStore = createNiceMock(TimelineMetricStore.class);
expect(metricStore.putMetrics(anyObject(TimelineMetrics.class)))
.andReturn(new TimelinePutResponse());
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
index 28bb75e..2f2b0b5 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/discovery/TestMetadataManager.java
@@ -160,7 +160,7 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest {
byte[] uuid = metadataManager.getUuid(timelineMetric, true);
Assert.assertNotNull(uuid);
- Assert.assertEquals(uuid.length, 20);
+ Assert.assertEquals(uuid.length, 32);
byte[] uuidWithoutHost = metadataManager.getUuid(new TimelineClusterMetric(timelineMetric.getMetricName(), timelineMetric.getAppId(), timelineMetric.getInstanceId(), -1), true);
Assert.assertNotNull(uuidWithoutHost);
diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
similarity index 59%
rename from ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java
rename to ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
index a25310b..e4018bb 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/MetricUuidGenStrategyTest.java
+++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/ambari/metrics/core/timeline/uuid/TimelineMetricUuidManagerTest.java
@@ -20,7 +20,6 @@ package org.apache.ambari.metrics.core.timeline.uuid;
import org.apache.ambari.metrics.core.timeline.aggregators.TimelineClusterMetric;
import org.junit.Assert;
-import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -38,30 +37,39 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-public class MetricUuidGenStrategyTest {
+public class TimelineMetricUuidManagerTest {
- private static List<String> apps = Arrays.asList("namenode",
- "datanode", "hbase_master", "hbase_regionserver", "kafka_broker", "nimbus", "ams-hbase",
+ private List<String> apps = Arrays.asList("namenode",
+ "datanode", "master_hbase", "slave_hbase", "kafka_broker", "nimbus", "ams-hbase",
"accumulo", "nodemanager", "resourcemanager", "ambari_server", "HOST", "timeline_metric_store_watcher",
- "jobhistoryserver", "hiveserver2", "hivemetastore", "applicationhistoryserver", "amssmoketestfake", "llapdaemon");
+ "jobhistoryserver", "hiveserver2", "hivemetastore", "applicationhistoryserver", "amssmoketestfake");
- private static Map<String, Set<String>> metricSet = new HashMap<>();
-
- @BeforeClass
- public static void init() {
- metricSet = new HashMap<>(populateMetricWhitelistFromFile());
- }
+ private Map<String, Set<String>> metricSet = new HashMap<>(populateMetricWhitelistFromFile());
@Test
- @Ignore
- public void testHashBasedUuid() throws SQLException {
- testMetricCollisionsForUuidGenStrategy(new HashBasedUuidGenStrategy(), 16);
+ @Ignore("Collisions possible")
+ public void testHashBasedUuidForMetricName() throws SQLException {
+
+ MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
+ Map<String, TimelineClusterMetric> uuids = new HashMap<>();
+ for (String app : metricSet.keySet()) {
+ Set<String> metrics = metricSet.get(app);
+ for (String metric : metrics) {
+ TimelineClusterMetric timelineClusterMetric = new TimelineClusterMetric(metric, app, null, -1l);
+ byte[] uuid = strategy.computeUuid(timelineClusterMetric, 16);
+ Assert.assertNotNull(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
+ Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(timelineClusterMetric));
+ uuids.put(uuidStr, timelineClusterMetric);
+ }
+ }
}
@Test
- @Ignore
- public void testHashBasedUuidForAppIds() throws SQLException {
+ public void testHaseBasedUuidForAppIds() throws SQLException {
+
MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
Map<String, TimelineClusterMetric> uuids = new HashMap<>();
for (String app : metricSet.keySet()) {
@@ -74,64 +82,53 @@ public class MetricUuidGenStrategyTest {
}
@Test
- @Ignore
public void testHashBasedUuidForHostnames() throws SQLException {
- testHostCollisionsForUuidGenStrategy(new HashBasedUuidGenStrategy(), 16);
- }
-
-
- @Test
- public void testMD5BasedUuid() throws SQLException {
- testMetricCollisionsForUuidGenStrategy(new MD5UuidGenStrategy(), 16);
-
- }
-
- @Test
- public void testMD5BasedUuidForHostnames() throws SQLException {
- testHostCollisionsForUuidGenStrategy(new MD5UuidGenStrategy(), 16);
- }
+ MetricUuidGenStrategy strategy = new HashBasedUuidGenStrategy();
+ Map<String, String> uuids = new HashMap<>();
- @Test
- public void testMD5ConsistentHashing() throws SQLException, InterruptedException {
- testConsistencyForUuidGenStrategy(new MD5UuidGenStrategy(), 16);
- }
+ List<String> hosts = new ArrayList<>();
+ String hostPrefix = "TestHost.";
+ String hostSuffix = ".ambari.apache.org";
+ for (int i=0; i<=2000; i++) {
+ hosts.add(hostPrefix + i + hostSuffix);
+ }
- @Test
- public void testMurmur3HashUuid() throws SQLException {
- testMetricCollisionsForUuidGenStrategy(new Murmur3HashUuidGenStrategy(), 16);
+ for (String host : hosts) {
+ byte[] uuid = strategy.computeUuid(host, 16);
+ Assert.assertNotNull(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
+ Assert.assertFalse(uuids.containsKey(uuidStr));
+ uuids.put(uuidStr, host);
+ }
}
- @Test
- public void testMurmur3HashingBasedUuidForHostnames() throws SQLException {
- testHostCollisionsForUuidGenStrategy(new Murmur3HashUuidGenStrategy(), 4);
- }
@Test
- public void testMurmur3ConsistentHashing() throws SQLException, InterruptedException {
- testConsistencyForUuidGenStrategy(new Murmur3HashUuidGenStrategy(), 4);
- }
+ public void testRandomUuidForWhitelistedMetrics() throws SQLException {
- private void testMetricCollisionsForUuidGenStrategy(MetricUuidGenStrategy strategy, int uuidLength) {
- Map<TimelineMetricUuid, TimelineClusterMetric> uuids = new HashMap<>();
+ MetricUuidGenStrategy strategy = new MD5UuidGenStrategy();
+ Map<String, String> uuids = new HashMap<>();
for (String app : metricSet.keySet()) {
Set<String> metrics = metricSet.get(app);
- for (String m : metrics) {
- TimelineClusterMetric metric = new TimelineClusterMetric(m, app, null, -1l);
- byte[] uuid = strategy.computeUuid(metric, uuidLength);
+ for (String metric : metrics) {
+ byte[] uuid = strategy.computeUuid(new TimelineClusterMetric(metric, app, null, -1l), 16);
Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == uuidLength);
- TimelineMetricUuid uuidStr = new TimelineMetricUuid(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
Assert.assertFalse(uuids.containsKey(uuidStr) && !uuids.containsValue(metric));
uuids.put(uuidStr, metric);
}
}
}
+ @Test
+ public void testRandomUuidForHostnames() throws SQLException {
- private void testHostCollisionsForUuidGenStrategy(MetricUuidGenStrategy strategy, int uuidLength) {
- Map<TimelineMetricUuid, String> uuids = new HashMap<>();
+ MetricUuidGenStrategy strategy = new MD5UuidGenStrategy();
+ Map<String, String> uuids = new HashMap<>();
List<String> hosts = new ArrayList<>();
String hostPrefix = "TestHost.";
@@ -141,33 +138,40 @@ public class MetricUuidGenStrategyTest {
hosts.add(hostPrefix + i + hostSuffix);
}
+ int numC = 0;
for (String host : hosts) {
- byte[] uuid = strategy.computeUuid(host, uuidLength);
+ byte[] uuid = strategy.computeUuid(host, 16);
Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == uuidLength);
- TimelineMetricUuid uuidStr = new TimelineMetricUuid(uuid);
+ Assert.assertTrue(uuid.length == 16);
+ String uuidStr = new String(uuid);
Assert.assertFalse(uuids.containsKey(uuidStr));
uuids.put(uuidStr, host);
}
}
- private void testConsistencyForUuidGenStrategy(MetricUuidGenStrategy strategy, int length) throws InterruptedException {
+
+ @Test
+ public void testConsistentHashing() throws SQLException, InterruptedException {
+
+ MetricUuidGenStrategy strategy = new MD5UuidGenStrategy();
String key = "TestString";
- byte[] uuid = strategy.computeUuid(key, length);
+ byte[] uuid = strategy.computeUuid(key, 16);
Assert.assertNotNull(uuid);
- Assert.assertTrue(uuid.length == length);
+ Assert.assertTrue(uuid.length == 16);
for (int i = 0; i<100; i++) {
- byte[] uuid2 = strategy.computeUuid(key, length);
+ byte[] uuid2 = strategy.computeUuid(key, 16);
Assert.assertNotNull(uuid2);
- Assert.assertTrue(uuid2.length == length);
+ Assert.assertTrue(uuid2.length == 16);
Assert.assertArrayEquals(uuid, uuid2);
Thread.sleep(10);
}
}
- private static Map<String, Set<String>> populateMetricWhitelistFromFile() {
+
+ public Map<String, Set<String>> populateMetricWhitelistFromFile() {
+
Map<String, Set<String>> metricSet = new HashMap<String, Set<String>>();
FileInputStream fstream = null;
@@ -203,7 +207,7 @@ public class MetricUuidGenStrategyTest {
}
}
metricsForApp.add("live_hosts");
- if (appId.startsWith("hbase")) {
+ if (appId.equals("master_hbase") || appId.equals("slave_hbase")) {
hbaseMetrics.addAll(metricsForApp);
} else {
metricSet.put(appId, metricsForApp);
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
index a111764..b7a55ae 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog270.java
@@ -1601,25 +1601,10 @@ public class UpgradeCatalog270 extends AbstractUpgradeCatalog {
if (clusters != null) {
Map<String, Cluster> clusterMap = clusters.getClusters();
+ ConfigHelper configHelper = injector.getInstance(ConfigHelper.class);
if (clusterMap != null && !clusterMap.isEmpty()) {
for (final Cluster cluster : clusterMap.values()) {
- Map<String, String> newProperties = new HashMap<>();
- newProperties.put("timeline.metrics.service.default.result.limit", "5760");
-
- Config config = cluster.getDesiredConfigByType("ams-site");
- if (config != null) {
- Map<String, String> oldAmsSite = config.getProperties();
- if (MapUtils.isNotEmpty(oldAmsSite)) {
- int oldTtl = Integer.parseInt(oldAmsSite.get("timeline.container-metrics.ttl"));
- if (oldTtl > 14 * 86400) {
- newProperties.put("timeline.container-metrics.ttl", "1209600");
- }
- }
- }
- Set<String> removeProperties = Sets.newHashSet("timeline.metrics.host.aggregate.splitpoints",
- "timeline.metrics.cluster.aggregate.splitpoints");
- updateConfigurationPropertiesForCluster(cluster, "ams-site", newProperties, removeProperties, true, true);
-
+ updateConfigurationPropertiesForCluster(cluster, "ams-site", Collections.singletonMap("timeline.metrics.service.default.result.limit", "5760"), true, true);
}
}
}
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
index e7eb3cb..0ffbf8a 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/configuration/ams-site.xml
@@ -541,6 +541,60 @@
<on-ambari-upgrade add="true"/>
</property>
<property>
+ <name>timeline.metrics.host.aggregate.splitpoints</name>
+ <value> </value>
+ <description>
+ Pre-split regions using the split points corresponding to this property
+ for the precision table that stores seconds aggregate data.
+ </description>
+ <depends-on>
+ <property>
+ <type>ams-hbase-site</type>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ </property>
+ <property>
+ <type>ams-hbase-site</type>
+ <name>hbase.hregion.memstore.flush.size</name>
+ </property>
+ <property>
+ <type>ams-hbase-env</type>
+ <name>hbase_master_heapsize</name>
+ </property>
+ <property>
+ <type>ams-hbase-env</type>
+ <name>hbase_regionserver_heapsize</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
+ <name>timeline.metrics.cluster.aggregate.splitpoints</name>
+ <value> </value>
+ <description>
+ Pre-split regions using the split points corresponding to this property
+ for the aggregate table that stores seconds aggregate data across hosts.
+ </description>
+ <depends-on>
+ <property>
+ <type>ams-hbase-site</type>
+ <name>hbase.regionserver.global.memstore.upperLimit</name>
+ </property>
+ <property>
+ <type>ams-hbase-site</type>
+ <name>hbase.hregion.memstore.flush.size</name>
+ </property>
+ <property>
+ <type>ams-hbase-env</type>
+ <name>hbase_master_heapsize</name>
+ </property>
+ <property>
+ <type>ams-hbase-env</type>
+ <name>hbase_regionserver_heapsize</name>
+ </property>
+ </depends-on>
+ <on-ambari-upgrade add="true"/>
+ </property>
+ <property>
<name>timeline.metrics.sink.report.interval</name>
<value>60</value>
<description>
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/service-metrics/KAFKA.txt b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/files/service-metrics/KAFKA.txt
old mode 100644
new mode 100755
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
index d0ee66e..73341f1 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/ams.py
@@ -226,35 +226,9 @@ def ams(name=None, action=None):
recursive_ownership = True
)
- new_ams_site = {}
- new_ams_site.update(params.config['configurations']['ams-site'])
- if params.clusterHostInfoDict:
- master_components = []
- slave_components = []
- components = dict(params.clusterHostInfoDict).keys()
- known_slave_components = ["nodemanager", "metrics_monitor", "datanode", "hbase_regionserver"]
- for component in components:
- if component and component.endswith("_hosts"):
- component_name = component[:-6]
- elif component and component.endswith("_host"):
- component_name = component[:-5]
- else:
- continue
- if component_name in known_slave_components:
- slave_components.append(component_name)
- else:
- master_components.append(component_name)
-
- if slave_components:
- new_ams_site['timeline.metrics.initial.configured.slave.components'] = ",".join(slave_components)
- if master_components:
- if 'ambari_server' not in master_components:
- master_components.append('ambari_server')
- new_ams_site['timeline.metrics.initial.configured.master.components'] = ",".join(master_components)
-
XmlConfig("ams-site.xml",
conf_dir=params.ams_collector_conf_dir,
- configurations=new_ams_site,
+ configurations=params.config['configurations']['ams-site'],
configuration_attributes=params.config['configurationAttributes']['ams-site'],
owner=params.ams_user,
group=params.user_group
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
index 9424752..de0fcf1 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/scripts/params.py
@@ -393,7 +393,6 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-clusterHostInfoDict = config["clusterHostInfo"]
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
index 7deeae6..c78d48a 100644
--- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
+++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/service_advisor.py
@@ -33,6 +33,15 @@ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+#split points
+metricsDir = os.path.join(SCRIPT_DIR, 'package')
+print "METRICS_DIR=>" + str(metricsDir)
+serviceMetricsDir = os.path.join(metricsDir, 'files', 'service-metrics')
+customServiceMetricsDir = os.path.join(SCRIPT_DIR, '../../../dashboards/service-metrics')
+sys.path.append(os.path.join(metricsDir, 'scripts'))
+
+from split_points import FindSplitPointsForAMSRegions
+
try:
with open(PARENT_FILE, 'rb') as fp:
service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
@@ -430,6 +439,19 @@ class AMBARI_METRICSRecommender(service_advisor.ServiceAdvisor):
if not ams_hbase_env:
ams_hbase_env = configurations["ams-hbase-env"]["properties"]
+ split_point_finder = FindSplitPointsForAMSRegions(
+ ams_hbase_site, ams_hbase_env, serviceMetricsDir, customServiceMetricsDir, operatingMode, servicesList)
+
+ result = split_point_finder.get_split_points()
+ precision_splits = ' '
+ aggregate_splits = ' '
+ if result.precision:
+ precision_splits = result.precision
+ if result.aggregate:
+ aggregate_splits = result.aggregate
+ putAmsSiteProperty("timeline.metrics.host.aggregate.splitpoints", ','.join(precision_splits))
+ putAmsSiteProperty("timeline.metrics.cluster.aggregate.splitpoints", ','.join(aggregate_splits))
+
component_grafana_exists = False
for service in services['services']:
if 'components' in service:
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
index ad6d435..2268694 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/upgrade/UpgradeCatalog270Test.java
@@ -1340,15 +1340,11 @@ public class UpgradeCatalog270Test {
Map<String, String> oldProperties = new HashMap<String, String>() {
{
put("timeline.metrics.service.default.result.limit", "15840");
- put("timeline.container-metrics.ttl", "2592000");
- put("timeline.metrics.cluster.aggregate.splitpoints", "cpu_user,mem_free");
- put("timeline.metrics.host.aggregate.splitpoints", "kafka.metric,nimbus.metric");
}
};
Map<String, String> newProperties = new HashMap<String, String>() {
{
put("timeline.metrics.service.default.result.limit", "5760");
- put("timeline.container-metrics.ttl", "1209600");
}
};
diff --git a/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
index ef2ac04..a97866b 100644
--- a/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
+++ b/ambari-server/src/test/python/common-services/AMBARI_METRICS/test_service_advisor.py
@@ -154,6 +154,8 @@ class TestAMBARI_METRICS010ServiceAdvisor(TestCase):
'hbase.unsafe.stream.capability.enforce': 'true'}},
'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
'timeline.metrics.cache.size': '100',
+ 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
'timeline.metrics.service.handler.thread.count': '20',
'timeline.metrics.service.operation.mode': 'distributed',
'timeline.metrics.service.watcher.disabled': 'true',
@@ -219,6 +221,8 @@ class TestAMBARI_METRICS010ServiceAdvisor(TestCase):
'hbase.unsafe.stream.capability.enforce': 'true'}},
'ams-site': {'properties': {'timeline.metrics.cache.commit.interval': '10',
'timeline.metrics.cache.size': '100',
+ 'timeline.metrics.cluster.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
+ 'timeline.metrics.host.aggregate.splitpoints': 'master.Balancer.BalancerCluster_95th_percentile',
'timeline.metrics.service.handler.thread.count': '20',
'timeline.metrics.service.operation.mode': 'distributed',
'timeline.metrics.service.watcher.disabled': 'true',
diff --git a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
index fc4d79d..58418f9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/AMBARI_METRICS/test_metrics_collector.py
@@ -147,7 +147,7 @@ class TestMetricsCollector(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/ambari-metrics-collector/conf',
configurations = self.getConfig()['configurations']['ams-site'],
- configuration_attributes = self.getConfig()['configurationAttributes']['ams-site']
+ configuration_attributes = self.getConfig()['configurationAttributes']['ams-hbase-site']
)
self.assertResourceCalled('XmlConfig', 'ssl-server.xml',
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 69bd8d8..220cc72 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -1076,9 +1076,7 @@
"timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier": "1",
"timeline.metrics.daily.aggregator.minute.interval": "86400",
"timeline.metrics.cluster.aggregator.minute.interval": "120",
- "timeline.metrics.host.aggregator.hourly.interval": "3600",
- "timeline.metrics.initial.configured.master.components": "drpc_server,hive_server,resourcemanager,all,webhcat_server,snamenode,storm_ui_server,falcon_server,namenode,ganglia_server,logviewer_server,hive_metastore,nimbus,zookeeper_server,historyserver,hbase_master,oozie_server,metrics_collector,supervisor,ganglia_monitor,hive_mysql,ambari_server",
- "timeline.metrics.initial.configured.slave.components": "hbase_regionserver,datanode,nodemanager"
+ "timeline.metrics.host.aggregator.hourly.interval": "3600"
},
"ams-grafana-env": {
"metrics_grafana_log_dir": "/var/log/ambari-metrics-grafana",
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
index 92cd24a..688c1c7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default_ams_embedded.json
@@ -994,8 +994,7 @@
"timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier": "1",
"timeline.metrics.daily.aggregator.minute.interval": "86400",
"timeline.metrics.cluster.aggregator.minute.interval": "120",
- "timeline.metrics.host.aggregator.hourly.interval": "3600",
- "timeline.metrics.initial.configured.master.components": "snamenode,nm,drpc_server,hive_server,resourcemanager,all,slave,webhcat_server,ganglia_server,storm_ui_server,falcon_server,hs,hive_metastore,logviewer_server,nimbus,zookeeper_server,hbase_rs,namenode,hbase_master,metrics_collector,ambari_server,supervisor,ganglia_monitor,hive_mysql"
+ "timeline.metrics.host.aggregator.hourly.interval": "3600"
}
},
"configurationAttributes": {
--
To stop receiving notification emails like this one, please contact
avijayan@apache.org.